{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('FE_train_LGBM.csv')\n",
    "test = pd.read_csv('FE_test_LGBM.csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\ipykernel_launcher.py:3: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "train['source'] = 'train'\n",
    "test['source'] = 'test'\n",
    "data = pd.concat([train,test],ignore_index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "\n",
    "data['msno_id'] = le.fit_transform(data['msno'])\n",
    "data['sid'] = le.fit_transform(data['song_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\pandas\\core\\frame.py:3940: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    }
   ],
   "source": [
    "train = data.loc[data['source']=='train']\n",
    "test = data.loc[data['source']=='test']\n",
    "train.drop(['source','id'],axis=1,inplace=True)\n",
    "test.drop(['source','target'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train= train.drop(['msno','song_id','target'],axis =1)\n",
    "y_train = train['target']\n",
    "X_test = test.drop(['id','msno','song_id'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>artist_name_counts</th>\n",
       "      <th>bd</th>\n",
       "      <th>city</th>\n",
       "      <th>expiration_date_Ym</th>\n",
       "      <th>expiration_date_month</th>\n",
       "      <th>expiration_date_year</th>\n",
       "      <th>gender</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>genre_ids_counts</th>\n",
       "      <th>...</th>\n",
       "      <th>msno_counts</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>song_id_counts</th>\n",
       "      <th>song_length_s</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_type</th>\n",
       "      <th>use_days</th>\n",
       "      <th>msno_id</th>\n",
       "      <th>sid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>346</td>\n",
       "      <td>1432.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0</td>\n",
       "      <td>111</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>101</td>\n",
       "      <td>155309.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7075</td>\n",
       "      <td>2</td>\n",
       "      <td>250</td>\n",
       "      <td>206.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>2103</td>\n",
       "      <td>9176</td>\n",
       "      <td>86884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3029</td>\n",
       "      <td>464160.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>11</td>\n",
       "      <td>110</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>244784.0</td>\n",
       "      <td>...</td>\n",
       "      <td>730</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>284.0</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2301</td>\n",
       "      <td>19273</td>\n",
       "      <td>260594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2060</td>\n",
       "      <td>458.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>11</td>\n",
       "      <td>110</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>244784.0</td>\n",
       "      <td>...</td>\n",
       "      <td>730</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>225.0</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2301</td>\n",
       "      <td>19273</td>\n",
       "      <td>140755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2164</td>\n",
       "      <td>1.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>11</td>\n",
       "      <td>110</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130.0</td>\n",
       "      <td>...</td>\n",
       "      <td>730</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>255.0</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2301</td>\n",
       "      <td>19273</td>\n",
       "      <td>27577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>444</td>\n",
       "      <td>527.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0</td>\n",
       "      <td>111</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>44343.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7075</td>\n",
       "      <td>2</td>\n",
       "      <td>474</td>\n",
       "      <td>187.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>2103</td>\n",
       "      <td>9176</td>\n",
       "      <td>38706</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   artist_name  artist_name_counts    bd  city  expiration_date_Ym  \\\n",
       "0          346              1432.0  27.0     0                 111   \n",
       "1         3029            464160.0  24.0    11                 110   \n",
       "2         2060               458.0  24.0    11                 110   \n",
       "3         2164                 1.0  24.0    11                 110   \n",
       "4          444               527.0  27.0     0                 111   \n",
       "\n",
       "   expiration_date_month  expiration_date_year  gender  genre_ids  \\\n",
       "0                      9                    13       2        101   \n",
       "1                      8                    13       0         27   \n",
       "2                      8                    13       0         27   \n",
       "3                      8                    13       0          1   \n",
       "4                      9                    13       2          0   \n",
       "\n",
       "   genre_ids_counts  ...  msno_counts  registered_via  song_id_counts  \\\n",
       "0          155309.0  ...         7075               2             250   \n",
       "1          244784.0  ...          730               3               1   \n",
       "2          244784.0  ...          730               3               6   \n",
       "3             130.0  ...          730               3               1   \n",
       "4           44343.0  ...         7075               2             474   \n",
       "\n",
       "   song_length_s  source_screen_name  source_system_tab  source_type  \\\n",
       "0          206.0                   7                  1            6   \n",
       "1          284.0                   8                  3            4   \n",
       "2          225.0                   8                  3            4   \n",
       "3          255.0                   8                  3            4   \n",
       "4          187.0                   7                  1            6   \n",
       "\n",
       "   use_days  msno_id     sid  \n",
       "0      2103     9176   86884  \n",
       "1      2301    19273  260594  \n",
       "2      2301    19273  140755  \n",
       "3      2301    19273   27577  \n",
       "4      2103     9176   38706  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['artist_name', 'artist_name_counts', 'bd', 'city', 'expiration_date_Ym',\n",
       "       'expiration_date_month', 'expiration_date_year', 'gender', 'genre_ids',\n",
       "       'genre_ids_counts', 'language', 'msno_counts', 'registered_via',\n",
       "       'song_id_counts', 'song_length_s', 'source_screen_name',\n",
       "       'source_system_tab', 'source_type', 'use_days', 'msno_id', 'sid'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "196.7311466666667"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_train)*2/3*0.014/350"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调参"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_ROUNDS =3000\n",
    "def get_n_estimators(params , X, y , early_stopping_rounds=10):\n",
    "    lgbm_params = params.copy()\n",
    "     \n",
    "    lgbmtrain = lgbm.Dataset(X, y )\n",
    "\n",
    "    cv_result = lgbm.cv(lgbm_params , lgbmtrain , num_boost_round=MAX_ROUNDS , nfold=3,  metrics='auc' , early_stopping_rounds=early_stopping_rounds,seed=3 )\n",
    "     \n",
    "    print('best n_estimators:' , len(cv_result['auc-mean']))\n",
    "    print('best cv score:' , cv_result['auc-mean'][-1])\n",
    "     \n",
    "    return len(cv_result['auc-mean'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## min_child_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] min_child_samples=10 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=10, score=0.782, total= 5.9min\n",
      "[CV] min_child_samples=10 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.9min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=10, score=0.783, total= 6.1min\n",
      "[CV] min_child_samples=10 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed: 12.0min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=10, score=0.783, total= 5.9min\n",
      "[CV] min_child_samples=20 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=20, score=0.783, total= 5.5min\n",
      "[CV] min_child_samples=20 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=20, score=0.783, total= 5.4min\n",
      "[CV] min_child_samples=20 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=20, score=0.782, total= 5.7min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed: 34.5min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=23, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      categorical_feature=[0, 3, 4, 5, 6, 7, 8,\n",
       "                                                           10, 12, 15, 16, 17,\n",
       "                                                           19, 20],\n",
       "                                      class_weight=None, colsample_bytree=0.3,\n",
       "                                      importance_type='split',\n",
       "                                      is_unbalance=True, learning_rate=0.1,\n",
       "                                      max_depth=10, min_child_sampl...\n",
       "                                      min_split_gain=0.0, n_estimators=500,\n",
       "                                      n_jobs=4, num_leaves=700,\n",
       "                                      objective='binary', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.9,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0, verbosity=5),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'min_child_samples': range(10, 30, 10)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='roc_auc', verbose=3)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':500,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.3,\n",
    "          'verbosity':5\n",
    "         }\n",
    "\n",
    "lg =LGBMClassifier(silent = False, **params)\n",
    "\n",
    "num_leaves_s = range(10, 30, 10)\n",
    "param_leaves =dict(min_child_samples = num_leaves_s)\n",
    "\n",
    "grid = GridSearchCV(lg, param_leaves, scoring = 'roc_auc', cv = kfold, verbose = 3, refit = False )\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD4CAYAAAApWAtMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hUddrG8e+TioD0oEgQkF4FDEVKorxUlaKrLui6rA0bStl11XWr67quugRRLKgrdkQsgIoUZQkgIAFBelUBQUBpIkh93j/m4I6RkIGQTMr9ua65MvM7v3PmOYPOzSl5MHdHREQkN2KiXYCIiBR+ChMREck1hYmIiOSawkRERHJNYSIiIrkWF+0CoqVSpUpeo0aNaJchIlKozJ8//xt3T8o6XmzDpEaNGmRmZka7DBGRQsXMvjzWuE5ziYhIrilMREQk1xQmIiKSawoTERHJNYWJiIjkmsJERERyTWEiIiK5pjA5Qe9+tol3Pv0Kte4XEfkfhckJenP+Rga9vpDrX8hk08590S5HRKRAUJicoGf7teRPlzRk9tpv6ZKewctzvuTIER2liEjxpjA5QbExxvXtazJpUCrnVivLH99ZQp9n5vD5N99HuzQRkahRmJyksyuW5OXrW/PQL5qyfPNuug3L4Knpazl0+Ei0SxMRyXcKk1wwM65sWY2pQ9JIrZvEgxNXcOkTH7Ns0+5olyYikq8UJqfAGWVKMPKa8xhxVQs279pHz8dn8u/JK9l/6HC0SxMRyRcKk1PEzLi4aRWmDE6j57ln8dhHa7h4+Ezmf7kj2qWJiOS5iMLEzLqZ2UozW2Nmdx9jebqZLQweq8xsZ9iyh8xsqZktN7PhFlLSzN4zsxXBsgfD5p9tZtPM7FMz+8zMLgpbdk9Qw0oz6xppffmpfKkEhv6yGc9f25K9+w9x+VMf87cJS9l74FA0yxIRyVM5homZxQIjgO5AQ6CvmTUMn+Pug929mbs3Ax4D3grWbQu0A5oCjYGWQFqw2iPuXh9oDrQzs+7B+B+BMe7eHOgDPBFsq2HwuhHQDXjCzGIjqS8aLqxXmclD0rimTXWen/UFXdIzmLn6m2iXJSKSJyI5MmkFrHH3de5+ABgN9DrO/L7Aa8FzB0oACUAiEA9scfe97j4NINjmAiA5bJ0ywfOywKbgeS9gtLvvd/fPgTVBbSdaX74pnRjHfb0aM+am84mPjeFXz83l92MXsWvvwWiXJiJySkUSJlWBDWGvNwZjP2Nm1YGawEcA7j4bmAZsDh6T3H15lnXKAT2AD4OhvwK/MrONwPvA7TnUcSL19TezTDPL3LZtW/Z7fIq1qlmBiQM7cMsFtXhzwVd0Sp/OB0u+zrf3FxHJa5GEiR1jLLtf+e4DjHX3wwBmVhtoQOiooyrQ0cxSf9ywWRyho5jh7r4uGO4LjHL3ZOAi4CUzizlOHRHX5+4j3T3F3VOSkpKy2YW8USI+lru61eedW9tRqXQiN788n9teWcC27/bnax0iInkhkjDZCFQLe53M/049ZdWH/53iArgUmOPue9x9DzARaBO2fCSw2t2HhY1dD4yBH49sSgCVjlPHidQXdU2SyzJ+QDvu7FqPKcu20GnodN6cv1GNI0WkUIskTOYBdcysppklEAqM8VknmVk9oDwwO2x4PZBmZnFmFk/o4vvyYP79hK6JDMqyqfXA/wVzGhAKk23Be/Yxs0QzqwnUAT6JtL6CJD42htsurM37AztQu3JpfvvGIn7z/Dw27tgb7dJERE5KjmHi7oeAAcAkQkEwxt2Xmtl9ZtYzbGpfQhfIw/+KPRZYCywGFgGL3H2CmSUD9xK6+2pBcEvxDcE6vwVuNLNFhI5yfuMhSwkdsSwDPgBuc/fD2dV3ch9H/qpduTRv3HQ+f+3RkHlfbKdregYvzv5CjSNFpNCx4np6JSUlxTMzM6Ndxo82bN/LH95ezIzV39CyRnke/EVTaiWVjnZZIiI/YWbz3T0l67h+A76AqFahJC9e14qHL2/Kyq+/o/ujM3jiv2s4qMaRIlIIKEwKEDPjipRqTP1tGh3rVeahD1bSe8Qslny1K9qliYgcl8KkAKp8egmeuuY8nry6BVt276fXiFk8PGkFPxxU40gRKZgUJgVY9yZVmDoklUubV2XEtLVcNHwGmV9sj3ZZIiI/ozAp4MqVTOCRK87lxetasf/gEa54ejZ/GbeEPfvVOFJECg6FSSGRWjeJyYNT6Xd+DV6c8yVd0zOYvir/WsKIiByPwqQQKZUYx197NuKNm84nMT6Gfv/5hN+OWcTOvQeiXZqIFHMKk0IopUYF3r+jAwMurM07C7+i09AMJi7eHO2yRKQYU5gUUiXiY/ld13qMH9COM8okcssrC7j5pfls3f1DtEsTkWJIYVLINTqrLONua8dd3erz0cqtdBo6nTcyN6hxpIjkK4VJERAXG8MtF9Ri4sAO1DvzdO4c+xm//s8nbNiuxpEikj8UJkVIraTSvN7/fP7eqxELvtxB12EZPD/rcw6rcaSI5DGFSRETE2Ncc34NJg1OpWWNCvxtwjKufHo2a7Z+F+3SRKQIU5gUUcnlSzLq2pYMvfJc1m7bw0WPzuTxj1arcaSI5AmFSRFmZlzWIpkpg9Po3OgMHpm8ip6Pq3GkiJx6CpNiIOn0REZc1YKnrzmPb/aEGkc+OFGNI0Xk1FGYFCNdG53J1MFpXN4imaemr6X7ozOYu+7baJclIkWAwqSYKVsynn9d3pSXr2/NwcNH+OXIOfzpnSV898PBaJcmIoWYwqSYal+nEpMHp3Jdu5q8PDfUOHLayq3RLktECimFSTFWMiGOP/doyJu3tKVUYhzXPj+PIa8vZMf3ahwpIidGYSK0OLs8797Rnjs61mb8ok10Gjqddz/bpJYsIhKxiMLEzLqZ2UozW2Nmdx9jebqZLQweq8xsZ9iyh8xsqZktN7PhFlLSzN4zsxXBsgdz2paZXRg2vtDMfjCz3sGyUWb2ediyZrn/aIqXxLhYhnSpx4Tb23NWudMY8Oqn9H9pPlvUOFJEImA5/e3TzGKBVUBnYCMwD+jr7suymX870NzdrzOztsDDQGqweCZwD/AJ0Nrdp5lZAvAh8IC7T8xuW1nGKwBrgGR332tmo4B33X1spDuekpLimZmZkU4vVg4dPsJzMz9n6JRVJMTF8MeLG3BlSjXMLNqliUiUmdl8d0/JOh7JkUkrYI27r3P3A8BooNdx5vcFXgueO1ACSAASgXhgi7vvdfdpAME2FwDJOWwr3OXARHdXJ8M8EBcbw01ptfhgUCoNqpThrjcXc/Wzc1n/rT5uETm2SMKkKrAh7PXGYOxnzKw6UBP4CMDdZwPTgM3BY5K7L8+yTjmgB6Gjk2y3lUUffh4y/zCzz4LTZInZ1NffzDLNLHPbNv2TtzmpWakUo29swz8ubcxnG3fRdVgGz81U40gR+blIwuRY5zay+zbpA4x198MAZlYbaEDoqKMq0NHMjp7ywsziCIXCcHdfd7xtha1TBWgCTAobvgeoD7QEKgB3Has4dx/p7inunpKUlJTNLki4mBjj6tbVmTIklfNrVeTv7y7jF09+zKotahwpIv8TSZhsBKqFvU4GNmUzN+sRw6XAHHff4+57gIlAm7DlI4HV7j4sgm0ddSXwtrv/+Ft27r7ZQ/YDzxM6NSenUJWyp/FcvxQe7dOML7/9nouHz+DRqas5cEiNI0UksjCZB9Qxs5rBxfI+wPisk8ysHlAemB02vB5IM7M4M4sH0oDlwfz7gbLAoAi3ddTPrqMERytY6Apxb2BJBPslJ8jM6NWsKlOHpNGtcRXSp66i5+MzWbRhZ84ri0iRlmOYuPshYACh00rLgTHuvtTM7jOznmFT+wKj/ae3h40F1gKLgUXAInefYGbJwL1AQ2BBcDvvDTlsCzOrQegoaXqWMl8xs8XB+1QC7s9pv+TkVSydyGN9m/PMr1PYsfcAlz4xiwfeX86+A2ocKVJc5XhrcFGlW4NPjd0/HOSf76/gtU/WU6NiSf55WVPOr1Ux2mWJSB7Jza3BItkqUyKef17WhFdvbI0DfZ+Zwx/eXsxuNY4UKVYUJnJKtK1ViQ8GpnJjh5qM/mQ9XYZm8NGKLdEuS0TyicJETpnTEmK59+KGvHVrO8qeFs91ozIZOPpTvt2zP9qliUgeU5jIKdesWjkm3N6eQZ3q8P7izXROz2Dcwq/UOFKkCFOYSJ5IiIthUKe6vHt7B6pVKMnA0Qu54YVMNu/aF+3SRCQPKEwkT9U783TeuqUtf7y4AbPWfkOXoRm8Onc9R9SSRaRIUZhInouNMW7ocA6TBqXSuGpZ/vD2Yq56dg5ffPN9tEsTkVNEYSL5pnrFUrx6Y2sevKwJS7/aTbdHM3gmY50aR4oUAQoTyVdmRp9WZzNlSBrta1fiH+8v57InZrHi693RLk1EckFhIlFxZtkSPPPrFB7r25yNO/ZxyfCZDJ2yiv2H1JJFpDBSmEjUmBk9zj2LKUPSuKRpFYZ/uJoej83k0/U7ol2aiJwghYlEXYVSCQzr05z//CaF7344xGVPfszf313G3gOHol2aiERIYSIFRsf6ZzB5cCpXtz6b52Z+TrdhM/h4zTfRLktEIqAwkQLl9BLx3N+7CaP7tyHG4Kpn53L3m5+xa58aR4oUZAoTKZDanFORDwalclPaOYzJ3EDnodOZvPTraJclItlQmEiBVSI+lnu6N+Cd29pRoVQC/V+az4BXF/CNGkeKFDgKEynwmiaXY/yA9vy2c10mL91Cp6HTefvTjWocKVKAKEykUEiIi+H2/6vDe3e0p2alUgx+fRHXjZrHpp1qHClSEChMpFCpc8bpjL25LX++pCFz1m2nS3oGL835Uo0jRaJMYSKFTmyMcV37mkwenEqzauX40ztL6PPMHNZt2xPt0kSKrYjCxMy6mdlKM1tjZncfY3m6mS0MHqvMbGfYsofMbKmZLTez4RZS0szeM7MVwbIHI9zW4bBl48PGa5rZXDNbbWavm1nCyX8kUlhUq1CSl65vxUO/aMryzbvp/ugMnpq+lkOHj0S7NJFiJ8cwMbNYYATQHWgI9DWzhuFz3H2wuzdz92bAY8BbwbptgXZAU6Ax0BJIC1Z7xN3rA82BdmbW/XjbCuw7uszde4aN/wtId/c6wA7g+hP6FKTQMjOubFmNqUPSSKubxIMTV9D7iVks26TGkSL5KZIjk1bAGndf5+4HgNFAr+PM7wu8Fjx3oASQACQC8cAWd9/r7tMAgm0uAJJz2NYxmZkBHYGxwdALQO8I9kuKkDPKlODpa87jiatb8PWuH+j5+Ez+PXmlGkeK5JNIwqQqsCHs9cZg7GfMrDpQE/gIwN1nA9OAzcFjkrsvz7JOOaAH8OHxthUoYWaZZjbHzI4GRkVgp7sfbeR0vPr6B+tnbtu27fh7LYWOmXFRkypMGZxGz2Zn8dhHa7h4+Ezmf6nGkSJ5LZIwsWOMZXfrTB9grLsfBjCz2kADQkcdVYGOZpb644bN4ggdeQx393XH21bgbHdPAa4ChplZrROpz91HunuKu6ckJSVlswtS2JUvlcDQK5sx6tqW7DtwmMuf+pi/TVjK9/vVOFIkr0QSJhuBamGvk4FN2cztw09PS10KzHH3Pe6+B5gItAlbPhJY7e7DItgW7r4p+LkO+C+h6y3fAOWCYMqpPilGLqhXmUmDU7mmTXWen/UFXYdlMGO1jkhF8kIkYTIPqBPcMZVA6Et+fNZJZlYPKA/MDhteD6SZWZyZxRO6+L48mH8/UBYYFMm2zKy8mSUGzysRurC/zEO/Bj0NuDyY2g8YF8F+STFQOjGO+3o1ZsxN55MQG8M1z33CnW8sYtdeNY4UOZVyDJPgWsQAYBKhIBjj7kvN7D4zC7+jqi8w2n/a42IssBZYDCwCFrn7BDNLBu4ldHfYguBW3xty2FYDINPMFhEKjwfdfVmw7C5giJmtIXQN5blIPwApHlrVrMD7Aztw6wW1eOvTr+iUPp0PlqhxpMipYsW1v1FKSopnZmZGuwyJgiVf7eL3Yz9j2ebdXNTkTP7asxGVTy8R7bJECgUzmx9cu/4J/Qa8FDuNq5Zl3IB23Nm1HlOXb6Xz0AzenK/GkSK5oTCRYik+NobbLqzN+3d0oHbl0vz2jUX0e34eG3fsjXZpIoWSwkSKtdqVS/PGTefzt56NyPxiO13TM3hx9hdqHClyghQmUuzFxBj92tZg0qBUWlQvz5/HLeXKp2ezVo0jRSKmMBEJVKtQkheva8UjV5zL6q176P7oDEZMW8NBNY4UyZHCRCSMmXH5eclMGZJKpwaVeXjSSnqPmMWSr3ZFuzSRAk1hInIMlU8vwRNXn8dTv2rBlt376TViFg99sIIfDqpxpMixKExEjqNb4yp8OCSNy5pX5Yn/ruWi4TOY98X2aJclUuAoTERyULZkPA9fcS4vXteK/QePcMVTs/nzuCXsUeNIkR8pTEQilFo3icmDU/lN2xq8NOdLuqZnMH2VGkeKgMJE5ISUSozjrz0bMfbm8ykRH0O//3zCkDEL2bn3QLRLE4kqhYnISTivegXeu6MDAy6szfiFm+g0dDrvL94c7bJEokZhInKSSsTH8ruu9Rg3oB1nli3Bra8s4OaX5rN19w/RLk0k3ylMRHKp0VlleefWdtzVrT4frdxKp6HTGZO5QY0jpVhRmIicAnGxMdxyQS0+GNiB+meW4fdjP+Oa5z5hw3Y1jpTiQWEicgqdk1Sa0f3b8Pfejfl0/Q66pGfw/KzPOazGkVLEKUxETrGYGOOaNtWZPCSN1udU4G8TlnHFUx+zZut30S5NJM8oTETySNVyp/H8b1qS/stzWffN91z06Ewe/2i1GkdKkaQwEclDZsalzZOZOiSNzo3O4JHJq+jx2EwWb1TjSClaFCYi+aBS6URGXNWCp685j+3fH6D3E7P458TlahwpRYbCRCQfdW10JlOGpHF5i2Senr6O7o/OYO66b6NdlkiuRRQmZtbNzFaa2Rozu/sYy9PNbGHwWGVmO8OWPWRmS81suZkNt5CSZvaema0Ilj2Y07bMrJmZzQ7mf2ZmvwxbZ5SZfR62XrPcfSwieafsafH86/KmvHJDaw4dOcIvR87hj+8s5rsfDka7NJGTZjn9YpWZxQKrgM7ARmAe0Nfdl2Uz/3agubtfZ2ZtgYeB1GDxTOAe4BOgtbtPM7ME4EPgAXefeJxt1QXc3Veb2VnAfKCBu+80s1HAu+4+NtIdT0lJ8czMzEini+SJvQcO8e/Jq/jPrM+pUqYE/7i0CRfWrxztskSyZWbz3T0l63gkRyatgDXuvs7dDwCjgV7Hmd8XeC147kAJIAFIBOKBLe6+192nAQTbXAAkH29b7r7K3VcHzzcBW4GkCOoXKbBKJsTxp0sa8uYtbSmVGMe1o+Yx+PWFbP9ejSOlcIkkTKoCG8JebwzGfsbMqgM1gY8A3H02MA3YHDwmufvyLOuUA3oQOjrJdltZlrUiFFBrw4b/EZz+SjezxGzq629mmWaWuW2bWodLwdHi7PK8e0d77vi/OkxYtInOQ6czYdEmtWSRQiOSMLFjjGX3X3gfYKy7HwYws9pAA0JHHVWBjmZ29JQXZhZH6MhjuLuvO962wtapArwEXOvuR2/YvweoD7QEKgB3Has4dx/p7inunpKUpIMaKVgS42IZ0rkuE25vT9Xyp3H7a59y44vz2aLGkVIIRBImG4FqYa+TgU3ZzO3D/05xAVwKzHH3Pe6+B5gItAlbPhJY7e7DItgWZlYGeA/4o7vPOTru7ps9ZD/wPKFTcyKFUoMqZXjrlrb84aL6zFi9jU5DpzP6k/U6SpECLZIwmQfUMbOawcXyPsD4rJPMrB5QHpgdNrweSDOzODOLB9KA5cH8+4GywKBIthW899vAi+7+Rpb5VYKfBvQGlkSwXyIFVlxsDP1TazFpUCoNq5Th7rcWc/Wzc1n/rRpHSsGUY5i4+yFgADCJUBCMcfelZnafmfUMm9oXGO0//evTWELXNRYDi4BF7j7BzJKBe4GGwILgdt4bctjWlYTuCvvNMW4BfsXMFgfvUwm4P+JPQKQAq1GpFK/d2IYHLm3CZxt30WXYdJ6dsU6NI6XAyfHW4KJKtwZLYbN51z7ufXsJH63YSrNq5Xjo8qbUPeP0aJclxUxubg0WkQKgStnTeK5fCo/2acb67Xu5ePgMHp26mgOH1DhSok9hIlKImBm9mlVlyuBUujeuQvrUUOPIRRt25ryySB5SmIgUQhVLJzK8b3Oe/XUKu/Yd5NInZvGP95ax74AaR0p0KExECrFODc9g8pBU+rQ6m2dmfE63RzOYvVaNIyX/KUxECrkyJeJ54NImvHpjawD6PjOHe95azG41jpR8pDARKSLa1qrEBwNT6Z96Dq/PW0+XoRl8uHxLtMuSYkJhIlKEnJYQyx8uasBbt7aj7GnxXP9CJne89inf7tkf7dKkiFOYiBRBzaqVY8Lt7RncqS4Tl2ym09DpjFv4lVqySJ5RmIgUUQlxMQzsVIf37uhA9YqlGDh6ITe8kMnmXfuiXZoUQQoTkSKu7hmn8+YtbfnjxQ2YtfYbOg/N4JW5X3JELVnkFFKYiBQDsTHGDR3OYfKgNJoml+Xet5dw1bNz+OKb76NdmhQRChORYuTsiiV55YbWPHhZE5Z+tZuuwzIYmbGWQ4fVkkVyR2EiUsyYGX1anc2UIWl0qJPEA++v4BdPfsyKr3dHuzQpxBQmIsXUmWVL8Myvz+Pxq5qzccc+Lhk+k6FTVrH/kFqyyIlTmIgUY2bGJU3PYuqQNHqcexbDP1zNJcNnsmD9jmiXJoWMwkREKF8qgfRfNuP537Rkz/5D/OLJj/n7u8vYe+BQtEuTQkJhIiI/urB+ZSYPTuXq1mfz3MzP6Tosg1lrvol2WVIIKExE5CdOLxHP/b2b8Hr/NsTFxHD1s3O5+83P2LVPjSMlewoTETmm1udUZOLADtycVos35m+k89DpTF76dbTLkgJKYSIi2SoRH8vd3evzzq3tqFg6kf4vzee2Vxew7Ts1jpSfiihMzKybma00szVmdvcxlqeb2cLgscrMdoYte8jMlprZcjMbbiElzew9M1sRLHswwm31M7PVwaNf2Ph5ZrY4qG+4mdnJfyQiklWT5LKMH9CO33Wpy5SlW+icPp23P92oxpHyoxzDxMxigRFAd6Ah0NfMGobPcffB7t7M3ZsBjwFvBeu2BdoBTYHGQEsgLVjtEXevDzQH2plZ9xy2VQH4C9AaaAX8xczKB9t6EugP1Ake3U7isxCR44iPjWFAxzq8P7A951QqxeDXF3HtqHl8tVONIyWyI5NWwBp3X+fuB4DRQK/jzO8LvBY8d6AEkAAkAvHAFnff6+7TAIJtLgCSc9hWV2CKu2939x3AFKCbmVUByrj7bA/9NelFoHcE+yUiJ6F25dN54+a2/KVHQ+au206XodN5aY4aRxZ3kYRJVWBD2OuNwdjPmFl1oCbwEYC7zwamAZuDxyR3X55lnXJAD+DD423rOHVUDZ5HUl9/M8s0s8xt27Zls7sikpPYGOPadjWZPDiV5meX50/vLKHPyDms27Yn2qVJlEQSJse6/pDdX0H6AGPd/TCAmdUGGhA66qgKdDSz1B83bBZH6MhjuLuvO962jlNHxPW5+0h3T3H3lKSkpGx2QUQiVa1CSV66vhUPXd6UFV/vptujM3jyv2ocWRxFEiYbgWphr5OBTdnM7cP/TksBXArMcfc97r4HmAi0CVs+Eljt7sMi2FZ2dWzkp6fIjlefiJxiZsaVKdWYOiSNC+sl8a8PVtD7iVks26TGkcVJJGEyD6hjZjXNLIHQl/z4rJPMrB5QHpgdNrweSDOzODOLJ3TxfXkw/36gLDAowm1NArqYWfngwnsXQqfNNgPfmVmb4C6uXwPjItgvETmFKpcpwdPXpPDk1S34etd+ej4+k0cmreSHg2ocWRzkGCbufggYQOjLfDkwxt2Xmtl9ZtYzbGpfYLT/9F7BscBaYDGwCFjk7hPMLBm4l9DdYQuC24BvON623H078HdC4TYPuC8YA7gFeBZYE7zfxIg/ARE5pbo3qcLUIan0alaVx6et4eLhM5j/5facV5RCzYrrfeIpKSmemZkZ7TJEirTpq7bxh7cWs2nXPvqdX4M7u9ajVGJctMuSXDCz+e6eknVcvwEvInkmrW4Skwan8us21Rn18Rd0Sc8gY5XupCyKFCYikqdKJ8bxt16NeePm80mMj+HX//mE372xiF171TiyKFGYiEi+aFmjAu/f0YFbL6jF259+Raf06XywZHO0y5JTRGEiIvmmRHwsv+9Wn3G3tSOpdCI3v7yAW16ez9bvfoh2aZJLChMRyXeNq5Zl3IB23Nm1Hh+u2ErnoRmMna/GkYWZwkREoiI+NobbLqzN+3d0oE7l0vzujUX0e34eG3fsjXZpchIUJiISVbUrl2bMTedzX69GzP9iO13SM3jh4y/UOLKQUZiISNTFxBi/Pr8GkwanklKjAn8Zv5Qrn57Nmq1qHFlYKExEpMBILl+SF65tyb+vOJfVW/dw0aMzGDFtDQfVOLLAU5iISIFiZvzivGSmDkmjU8PKPDxpJb0en8WSr3ZFuzQ5DoWJiBRISacn8sTV5/HUr1qwbc9+eo2Yxb8+WKHGkQWUwkRECrRujaswdXAalzWvypP/XctFj85g3hdqHFnQKExEpMArWzKeh684l5eub8WBw0e44qnZ/HncEvbsPxTt0iSgMBGRQqNDnSQmDUrl2nY1eGnOl3RNz+C/K7dGuyxBYSIihUypxDj+0qMRY29uy2kJsfzm+XkMGbOQHd8fiHZpxZrCREQKpfOql+e9O9pze8fajF+4ic7p03l/8Wa1ZIkShYmIFFqJcbH8tks9xg9oT5Wyp3HrKwu4+eX5bN2txpH5TWEiIoVew7PK8Patbbmne33+u3IbnYZOZ8y8DTpKyUcKExEpEuJiY7gprRYTB3agfpUy/P7Nz7jmuU/YsF2NI/ODwkREipRzkkoz+sY23N+7MQs37KRLegb/mfk5h9U4Mk9FFCZm1s3MVprZGjO7+xjL081sYfBYZWY7w5Y9ZGZLzWy5mQ23kJJm9p6ZrWko+8IAAA5mSURBVAiWPZhle1ea2bJg2avB2IVh77HQzH4ws97BslFm9nnYsma5+1hEpDCLiTF+1aY6kwen0vqcCtz37jKueOpjVm/5LtqlFVmW0zlFM4sFVgGdgY3APKCvuy/LZv7tQHN3v87M2gIPA6nB4pnAPcAnQGt3n2ZmCcCHwAPuPtHM6gBjgI7uvsPMKrv71izvUQFYAyS7+14zGwW86+5jI93xlJQUz8zMjHS6iBRS7s64hZv424SlfL//MLd3rM3NF9QiPlYnZk6Gmc1395Ss45F8mq2ANe6+zt0PAKOBXseZ3xd4LXjuQAkgAUgE4oEt7r7X3acBBNtcACQH69wIjHD3HcHyY/1G0uXARHfXyVAROS4zo3fzqkwZkkaXRmfw7ymr6PHYTBZvVOPIUymSMKkKbAh7vTEY+xkzqw7UBD4CcPfZwDRgc/CY5O7Ls6xTDuhB6OgEoC5Q18xmmdkcM+t2jLfqw/8C66h/mNlnwSm3xGzq629mmWaWuW3btuz3WESKnEqlE3n8qhaMvOY8duw9QK8RM/nnxOVqHHmKRBImdoyx7M6N9QHGuvthADOrDTQgdNRRFehoZkdPeWFmcYRCYbi7rwuG44A6wAWEjnKeDQLn6DpVgCbApLD3vQeoD7QEKgB3Has4dx/p7inunpKUlHS8fRaRIqpLozOZPDiNX7asxtPT19FtWAZz1n0b7bIKvUjCZCNQLex1MrApm7lZjxguBea4+x533wNMBNqELR8JrHb3YVneb5y7H3T3z4GVhMLlqCuBt9394NEBd9/sIfuB5wmdmhMROaayp8Xzz8ua8uoNrTni0GfkHO59ezHf/XAw55XlmCIJk3lAHTOrGVws7wOMzzrJzOoB5YHZYcPrgTQzizOzeCANWB7Mvx8oCwzKsql3gAuDOZUInfZaF7Y8/JrM0feuEvw0oDewJIL9EpFirm3tSnwwqAM3tK/Ja5+sp0t6BtNWqHHkycgxTNz9EDCA0Gml5cAYd19qZveZWc+wqX2B0f7T28PGAmuBxcAiYJG7TzCzZOBeoCGwILid94ZgnUnAt2a2jND1ljvd/VsAM6tB6ChpepYyXzGzxcH7VALuj/QDEJHirWRCHH+8pCFv3tKW0olxXDtqHoNGf8p2NY48ITneGlxU6dZgEclq/6HDPDFtLU/8dw1lSsTz156NuKRpFUInPQRyd2uwiEixkBgXy+DOdZlwe3uSy5/G7a99yo0vzufrXWocmROFiYhIFvXPLMNbt7bj3osaMHPNNjoPnc5rn6xX48jjUJiIiBxDbIxxY+o5fDAwlUZVy3DPW4u56pm5fPnt99EurUBSmIiIHEeNSqV49YY2PHBpE5Z8tYuuwzJ4dsY6NY7MQmEiIpKDmBjjqtZnM3lIKu1qVeL+95Zz2ZMfs/JrNY48SmEiIhKhKmVP49l+KQzv25wN2/dyyWMzGDZ1FQcOHYl2aVGnMBEROQFmRs9zz2LqkDQualKFYVNX0+OxmSzcsDPnlYswhYmIyEmoUCqBR/s057l+Kezad5DLnpjFP95bxr4DxbNxpMJERCQX/q/BGUwekkqfVmfzzIzP6Tosg4/XfhPtsvKdwkREJJfKlIjngUub8NqNbTCDq56Zyz1vLWZ3MWocqTARETlFzq9VkQ8GptI/9Rxen7eezkOnM3XZlmiXlS8UJiIip9BpCbH84aIGvH1rO8qXTOCGFzO547VP+XbP/miXlqcUJiIieeDcauUYP6A9QzrXZeKSzXQaOp1xC78qsi1ZFCYiInkkIS6GO/6vDu/d0YHqFUsxcPRCrn8hk00790W7tFNOYSIiksfqnnE6b97Slj9d0pDZa7+lS3oGr8z9kiNFqCWLwkREJB/ExhjXt6/JpEGpnFutLPe+vYS+z8zh82+KRuNIhYmISD46u2JJXr6+Nf/6RROWbd5Nt2EZjMxYy6HDhbsli8JERCSfmRm/bHk2U4ekkVo3iQfeX8FlT37M8s27o13aSVOYiIhEyRllSjDymvMYcVULNu3cR4/HZjJ08kr2Hyp8LVkUJiIiUWRmXNy0ClMGp9Hz3LMY/tEaLhk+kwXrd0S7tBOiMBERKQDKl0pg6C+b8fy1Lfl+/yF+8eTH3DdhGXsPHIp2aRGJKEzMrJuZrTSzNWZ29zGWp5vZwuCxysx2hi17yMyWmtlyMxtuISXN7D0zWxEsezDL9q40s2XBslfDxg+Hvc/4sPGaZjbXzFab2etmlnByH4eISHRdWK8ykwan8qvW1fnPrFDjyFlrCn7jyBzDxMxigRFAd6Ah0NfMGobPcffB7t7M3ZsBjwFvBeu2BdoBTYHGQEsgLVjtEXevDzQH2plZ92CdOsA9QDt3bwQMCnurfUffx917ho3/C0h39zrADuD6E/kQREQKktNLxPP33o15vX8b4mJiuPrZudw19jN27Su4jSMjOTJpBaxx93XufgAYDfQ6zvy+wGvBcwdKAAlAIhAPbHH3ve4+DSDY5gIgOVjnRmCEu+8Ilm89XnFmZkBHYGww9ALQO4L9EhEp0FqfU5GJAztwywW1GLtgI52HTmfS0q+jXdYxRRImVYENYa83BmM/Y2bVgZrARwDuPhuYBmwOHpPcfXmWdcoBPYAPg6G6QF0zm2Vmc8ysW9j0EmaWGYwfDYyKwE53P3pi8Xj19Q/Wz9y2bVsEuy4iEl0l4mO5q1t93rm1HRVLJ3LTS/O57ZUFbPuuYDWOjCRM7Bhj2fUA6AOMdffDAGZWG2hA6KijKtDRzFJ/3LBZHKGjmOHuvi4YjgPqABcQOsp5NggcgLPdPQW4ChhmZrVOpD53H+nuKe6ekpSUlN3+iogUOE2SyzJ+QDvu7FqPKcu20Dl9Om8t2FhgGkdGEiYbgWphr5OBTdnM7cP/TnEBXArMcfc97r4HmAi0CVs+Eljt7sOyvN84dz/o7p8DKwmFC+6+Kfi5Dvgvoest3wDlgmDKqT4RkUIrPjaG2y6szfsD23NOpVIMGbOIa0fN46sC0DgykjCZB9QJ7phKIBQY47NOMrN6QHlgdtjweiDNzOLMLJ7Qxfflwfz7gbL89AI7wDvAhcGcSoROe60zs/Jmlhg23g5Y5qFYngZcHqzfDxgXwX6JiBRKtSufzhs3t+WvPRryyefb6TJ0Oi/N/iKqjSNzDJPgWsQAYBKhIBjj7kvN7D4zC7+jqi8w2n96zDUWWAssBhYBi9x9gpklA/cSujtsQXCr7w3BOpOAb81sGaGQuNPdvyV0uizTzBYF4w+6+7JgnbuAIWa2htA1lOdO/KMQESk8YmOM37QLNY5sUb08fxq3lD4j57B2256o1GMF5XxbfktJSfHMzMxolyEikmvuztj5G/n7u8v44dARBnWqQ/8O5xAXe+p/L93M5gfXrn9CvwEvIlLImRlXpFRj6m/T6FivMg99sJLeT8xi6aZd+VaDwkREpIiofHoJnrrmPJ68ugVf79pPz8dn8fCkFfxwMO8bRypMRESKmO5NqjB1SCq9m1VlxLS1XDx8BvO/3J6n76kwEREpgsqVTODfV57LC9e14oeDR7j8qdn8dfxSvt+fN40jFSYiIkVYWt0kJg9Opd/5NXhh9hd0Sc9g5dffnfL3UZiIiBRxpRLj+GvPRrxx0/nUqlya5PKnnfL3iMt5ioiIFAUpNSrw4nWt8mTbOjIREZFcU5iIiEiuKUxERCTXFCYiIpJrChMREck1hYmIiOSawkRERHJNYSIiIrlWbP89EzPbBnx5kqtXIvTPBRcn2ufiQftc9OV2f6u7e1LWwWIbJrlhZpnH+sdhijLtc/GgfS768mp/dZpLRERyTWEiIiK5pjA5OSOjXUAUaJ+LB+1z0Zcn+6trJiIikms6MhERkVxTmIiISK4pTHJgZv8xs61mtiRsrIKZTTGz1cHP8tGs8VTLZp8fNrMVZvaZmb1tZuWiWeOpdqx9Dlv2OzNzM6sUjdryQnb7a2a3m9lKM1tqZg9Fq768kM1/183MbI6ZLTSzTDPLm385KkrMrJqZTTOz5cGf6cBg/JR/hylMcjYK6JZl7G7gQ3evA3wYvC5KRvHzfZ4CNHb3psAq4J78LiqPjeLn+4yZVQM6A+vzu6A8Noos+2tmFwK9gKbu3gh4JAp15aVR/PzP+CHgb+7eDPhz8LooOQT81t0bAG2A28ysIXnwHaYwyYG7ZwDbswz3Al4Inr8A9M7XovLYsfbZ3Se7+6Hg5RwgOd8Ly0PZ/DkDpAO/B4rUnSrZ7O8twIPuvj+YszXfC8tD2eyzA2WC52WBTflaVB5z983uviB4/h2wHKhKHnyHKUxOzhnuvhlCf1hA5SjXk9+uAyZGu4i8ZmY9ga/cfVG0a8kndYEOZjbXzKabWctoF5QPBgEPm9kGQkdiRe2I+0dmVgNoDswlD77DFCZyQszsXkKHzq9Eu5a8ZGYlgXsJnfooLuKA8oROh9wJjDEzi25Jee4WYLC7VwMGA89FuZ48YWalgTeBQe6+Oy/eQ2FycraYWRWA4GeROh2QHTPrB1wCXO1F/xeUagE1gUVm9gWh03oLzOzMqFaVtzYCb3nIJ8ARQk0Bi7J+wFvB8zeAInUBHsDM4gkFySvufnRfT/l3mMLk5Iwn9B8hwc9xUawlX5hZN+AuoKe77412PXnN3Re7e2V3r+HuNQh90bZw96+jXFpeegfoCGBmdYEEin433U1AWvC8I7A6irWccsGR5XPAcncfGrbo1H+Hubsex3kArwGbgYOEvlCuByoSugNidfCzQrTrzId9XgNsABYGj6eiXWde73OW5V8AlaJdZx7/GScALwNLgAVAx2jXmQ/73B6YDywidC3hvGjXeYr3uT2hmww+C/t/96K8+A5TOxUREck1neYSEZFcU5iIiEiuKUxERCTXFCYiIpJrChMREck1hYmIiOSawkRERHLt/wEP7l8nC7b8ygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, test_means)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] min_child_samples=30 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=30, score=0.782, total= 5.4min\n",
      "[CV] min_child_samples=30 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.4min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=30, score=0.783, total= 5.6min\n",
      "[CV] min_child_samples=30 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed: 11.1min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=30, score=0.782, total= 5.7min\n",
      "[CV] min_child_samples=50 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=50, score=0.782, total= 5.6min\n",
      "[CV] min_child_samples=50 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=50, score=0.783, total= 6.0min\n",
      "[CV] min_child_samples=50 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=50, score=0.782, total= 6.0min\n",
      "[CV] min_child_samples=70 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=70, score=0.781, total= 6.1min\n",
      "[CV] min_child_samples=70 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=70, score=0.782, total= 5.7min\n",
      "[CV] min_child_samples=70 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ min_child_samples=70, score=0.781, total= 6.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed: 52.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=23, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      categorical_feature=[0, 3, 4, 5, 6, 7, 8,\n",
       "                                                           10, 12, 15, 16, 17,\n",
       "                                                           19, 20],\n",
       "                                      class_weight=None, colsample_bytree=0.3,\n",
       "                                      importance_type='split',\n",
       "                                      is_unbalance=True, learning_rate=0.1,\n",
       "                                      max_depth=10, min_child_sampl...\n",
       "                                      min_split_gain=0.0, n_estimators=500,\n",
       "                                      n_jobs=4, num_leaves=700,\n",
       "                                      objective='binary', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.9,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0, verbosity=5),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'min_child_samples': range(30, 90, 20)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='roc_auc', verbose=3)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':500,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.3,\n",
    "          'verbosity':5\n",
    "         }\n",
    "\n",
    "lg =LGBMClassifier(silent = False, **params)\n",
    "\n",
    "num_leaves_s = range(30, 90, 20)\n",
    "param_leaves =dict(min_child_samples = num_leaves_s)\n",
    "\n",
    "grid = GridSearchCV(lg, param_leaves, scoring = 'roc_auc', cv = kfold, verbose = 3, refit = False )\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.782481  , 0.78223084, 0.78161111])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_[ 'mean_test_score' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU9b3/8dcnySQhLAlL2DKAKMgiaxIRsHXrhlZFDaC41YoivbX32t57e+29tz9b7eLSVqv1ChSp1g0laN2lLlRrDQiJoLLI5sKwBiGEEAIh+fz+mNFGkkjQJGeSvJ+PRx7mzDkz5z0nOO+Z78x8j7k7IiIiNSUEHUBEROKPykFERGpROYiISC0qBxERqUXlICIitSQFHaAxdOvWzY855pigY4iItCiFhYU73T2zrnWtohyOOeYYli1bFnQMEZEWxcw+rG+dhpVERKQWlYOIiNSichARkVpUDiIiUovKQUREalE5iIhILSoHERGppU2Xw9uREu56eR2bS/YHHUVEJK60ii/BfVEFGz7mty+u5XcvrWX8cV3Jyw4zYVhP0pLb9GEREcFaw8l+cnNz/Yt+Q3rTrnIWFEVYUBRh0679tE9O5NsjepGXHWZM/y6YWSOnFRGJD2ZW6O65da5r6+XwiepqZ+kHu1hQFOHZt7ey72AVfbukkZcd5oLsLPp0SWuktCIi8UHlcJTKDx7ihXe3saAowhsbPsYdxh7bhbzsMGcN70X7FA07iUjLp3L4EiK7y3miaDMLiiJ88HE5acmJnDmsF3k5WYzt35WEBA07iUjLpHJoBO5O4Ye7WVAU4ZkVW9l74BBZGe3IywmTl51Fv67tm3T/IiKNTeXQyPYfrOKvq7aRXxjh9fU7cYcxx3QhLyeLs4b3omNqqNmyiIh8USqHJrR1z34ejw07bSzeR2ooITrslB1m/HEadhKR+KVyaAbuzlubSlhQGOGpFVvYW3GI3umpXJAdJi8nTP9uGnYSkfiicmhmFZVVvLhqOwuKIry2tphqh5x+ncnLDnP2yF500rCTiMQBlUOAtpdW8MRbm8kvjLB+RxkpSQl864Se5OWE+cqAbiRq2ElEAqJyiAPuztuRPeTHhp327K+kZ6dUzs/OIi87zIDuHYKOKCJtjMohzhw4VMXLq3ewoDDC39YWU1XtjOqTQV5OmHNH9CY9TcNOItL0VA5xbMfeCp58awv5hRHe276X5KQEvjG0B5Oyw3x1YDeSEtv0xLki0oRUDi2Au7NySyn5hRGeXL6Z3eWVZHZM4YLRWeTlhDm+R8egI4pIK/Oly8HMJgC/BxKBOe5+82HrbwdOjy2mAd3dPSO27lbg20TPHfEi8G9AO2A+cBxQBTzt7tcfdpuTYtuc6O6f+8jfGsqhpoOHqnllzQ4WFEVYtGYHh6qdEeF08rLDnDuyN53bJwcdUURagS9VDmaWCKwFvgFEgKXAVHdfVc/2PwBGu/uVZjYeuA04Jbb6deAnwJvASe6+yMySgZeBX7n787Hb6Ag8CyQD17a1cqhpZ9kBnly+hQWFEVZtLSWUaHx9SA/yssOcOiiTkIadROQL+rxyaMj0omOA9e6+MXZj84CJQJ3lAEwFboj97kAq0Qd5A0LAdncvBxYBuPtBMysCwjVu4ybgVuA/GpCvVevWIYVpX+nPtK/0Z+WWPSwo3MyTyzfz/Lvb6NYhmfNGRYedhvTqFHRUEWlFGlIOWcCmGssR4KS6NjSzfkB/4BUAdy8ws0XAVqLl8Ad3X33YdTKAc4gOW2Fmo4E+7v6MmbX5cqjphN7pnNA7nZ+cNZi/vVfMgsII9xd8wJzX3+eE3p3Iyw4zcVRvunZICTqqiLRwDSmHur6lVd9Y1EVAvrtXAZjZAGAI/3xV8KKZneLur8XWJwGPAHe6+0YzSwBuB644Yiiz6cB0gL59+zbgbrQeocToJ5q+MbQHu/Yd5Knlm1lQtJkbn1nFr55bzRmDu5OXE+b0Qd1JTtKwk4gcvYa85zAO+Jm7fyu2/BMAd/91Hdu+BXzf3d+ILf8nkOruN8WW/x9Q4e63xpbnAmXu/q+x5XRgA1AWu8mewC7g3M9736E1v+dwNNZsK2VBYYQn3trCzrIDdGmfzMRRvcnLDnNC70465amIfMaXfUM6iegb0l8DNhN9Q/pid1952HaDgIVAf4/dqJldCFwNTCD6CuQF4A53f9rMfkH0VcVkd6+uZ99/A/6jLb8h/UUcqqrmtXXFLCjczIurtnOwqprBPTsyKSfMxFFZZHbUsJOIfMk3pN39kJldS/SBPxGY6+4rzexGYJm7PxXbdCowzz/bNvnAGcA7RIeiXogVQxj4H2ANUBR7RvsHd5/zxe6i1JSUmMAZg3twxuAelJQf5OkVW8gv2swvnl3Nr59fw+mDMsnLDnPGkO6kJCUGHVdE4pC+BNeGrNu+l/yiCE8UbWbH3gNkpIWYOLI3eTlhhmela9hJpI3RN6TlMw5VVfP6+p0sKNrMwpXbOHiomuN7dCAvO8z5o7Po3ik16Igi0gxUDlKvPfsreebt6Jfsij4qIcHg1OMzmZTTh68N6U5qSMNOIq2VykEaZENxGQsKIzxetJltpRV0Sk3i3NinnUb1ydCwk0gro3KQo1JV7byxYSf5hRFeeHcbBw5Vc1xme/JywlwwOkzPdA07ibQGKgf5wkorKnnu7a0sKIqw9IPdJBh8ZWAmk3LCfHNoDw07ibRgKgdpFO/v3MfjRREWFEbYsqeCjqlJnD2iN5Nyssju21nDTiItjMpBGlV1tbN448fkF0Z4/t1t7K+son+39kzKiX7aqXdGu6AjikgDqBykyZQdOMRz72wlvzDCm+/vwgxOPq4bk3LCfOuEnrRL1rCTSLxSOUiz+OjjchYURVhQFCGyez8dUpL49vBe5OWEOfEYDTuJxBuVgzSr6mrnzQ92kV8Y4bl3tlJ+sIp+XdPIyw5zQXYW4c5pQUcUEVQOEqB9Bw7xwrvbyC+MULDxYwDGHduVSTlhJgzrSfuUhswaLyJNQeUgcWHTrnKeeGsz+YURPtpVTlpyImcN70VedpiT+nchIUHDTiLNSeUgccXdWfbhbvKXRXj2na2UHThEuHM78rLD5GWH6dtVw04izUHlIHFr/8EqFq6MDjv9Y8NO3GFM/y5Myglz1vBedNCwk0iTUTlIi7ClZP+nw07v79xHu1AiZw7rSV5OmHHHdtWwk0gjUzlIi+LuFH1UQn5hhGfe3sLeikNkZbTjguws8rLDHNOtfdARRVoFlYO0WBWVVfx11XbyCyO8vq6Yaofcfp2jw04jetEpNRR0RJEWS+UgrcK2PRWxYadNbCjeR0pSAhOG9SQvO8zJA7qRqGEnkaOicpBWxd1ZEdlDfuEmnlq+hdKKQ/TslBoddsoJc1xmh6AjirQIKgdptSoqq3h59Q7yCzfx6trosNPovhlMyglz9ojepLfTsJNIfVQO0ibsKK3gL8ujn3Zau72M5KQEvjm0B3k5Yb46oBtJiQlBRxSJKyoHaVPcnXc3l5JfuIknV2yhpLyS7h1TOD87i0nZYQb26Bh0RJG4oHKQNuvAoSoWrdlBfmGERe8VU1XtjAynMyknzDkje5ORlhx0RJHAqBxEgOK9B3gyNuy0ZttekhMT+PrQ7uRlhzn1+EwNO0mbo3IQOczKLXvIL4zw5PIt7Np3kG4dUjh/dG/ycsIM7tkp6HgizULlIFKPg4eq+dt70WGnV9bs4FC1MyyrE5Oyw5w7Kosu7TXsJK2XykGkAT4uO8BTK7aQXxhh5ZZSQonGGYOjw06nD+5OSMNO0sqoHESO0uqtpSwojPCX5ZvZWXaQru2TmTgqi7ycLE7onR50PJFGoXIQ+YIqq6p5bW0x+YURXl69g4NV1Qzp1YlJOWEmjupNtw4pQUcU+cJUDiKNYPe+gzz9dnTY6e3IHpISjNMGdWdSThZnDO5BcpKGnaRlUTmINLK12/eyoDDC429tpnjvATqnhaLDTtlhhmV1wkyTAEr8UzmINJFDVdX8ff1O8gsjvLhqOwcPVTOoR8fosNPo3nTvmBp0RJF6qRxEmsGe8spPh52WbyohMcE4fVAmN503jF7p7YKOJ1KLykGkma3fUcaCoggPFHxIersQD0wbw7GaSlzizOeVg95BE2kCA7p34L8mDGbe9LFUVFYxeWYB727eE3QskQZTOYg0oWFZ6Tw2YxypoUSmzl7Mko0fBx1JpEFUDiJN7LjMDsyfMY7unVK4fO6bvLRqe9CRRI6oQeVgZhPM7D0zW29m19ex/nYzWx77WWtmJTXW3WpmK81stZndaVFpZvasma2Jrbu5xvY/MrNVZva2mb1sZv0a566KBKd3RjvmzxjPoJ4duebBQh4vigQdSeRzHbEczCwRuBs4ExgKTDWzoTW3cfcfuvsodx8F3AU8HrvueOBkYAQwDDgRODV2td+4+2BgNHCymZ0Zu/wtINfdRwD5wK1f7i6KxIcu7ZN5+OqxnNS/Cz96bAVzX38/6Egi9WrIK4cxwHp33+juB4F5wMTP2X4q8EjsdwdSgWQgBQgB29293N0XAcRuswgIx5YXuXt57PqLP7lcpDXokJLE3CtO5Fsn9ODGZ1bxu7++R2v4xKC0Pg0phyxgU43lSOyyWmJDQP2BVwDcvQBYBGyN/Sx099WHXScDOAd4uY6bnAY8X8++ppvZMjNbVlxc3IC7IRIfUkOJ3H1xNlNyw9z5ynpueGol1dUqCIkvSQ3Ypq55AOr7l3wRkO/uVQBmNgAYwj+f/b9oZqe4+2ux9UlEX2Xc6e4bP7NTs0uBXP45DPXZAO6zgdkQ/Z5DA+6HSNxISkzglrwRZKQlM/u1jZSUV/KbySM1P5PEjYaUQwToU2M5DGypZ9uLgO/XWD4fWOzuZQBm9jwwFngttn42sM7d76h5I2b2deB/gFPd/UADMoq0OGbGf581hM5pydzywhpKKyq555Ic2iUnBh1NpEHDSkuBgWbW38ySiRbAU4dvZGaDgM5AQY2LPwJONbMkMwsRfRWwOrb9L4B04LrDbmc0MAs41913HP1dEmlZvnfacfz6guG8uraYy+5dwp79lUFHEjlyObj7IeBaYCHRB/bH3H2lmd1oZufW2HQqMM8/++5aPrABeAdYAaxw96fNLEz0lcFQoCj2EdirYte5DegAzI9dXquIRFqbqWP6cvfF2ayIlHDhrAJ2lFYEHUnaOM2tJBJH/r6umGseKKRbhxQenHYSfbumBR1JWjHNrSTSQnx1YCYPXXUSe/ZXMmnmG6zZVhp0JGmjVA4icWZ0387MnzEOM5gys4DCD3cHHUnaIJWDSBw6vkdH8meMp0v7ZC6ds4RX1+q7PNK8VA4icapPlzTmzxhP/27tuer+pTy9or5PkIs0PpWDSBzL7JjCI9PHMqpPBv867y0eXPxh0JGkjVA5iMS59HYh/nzlSZw+qDv/+5d3uXvRes3HJE1O5SDSArRLTmTWZTmcN6o3ty18j18+u1rzMUmTasj0GSISB0KJCfxuyigy0pKZ8/r7lOyv5OYLhpOUqOd40vhUDiItSEKCccM5Q8lIC3HHS+vYs7+Su6aOJjWk+Zikcekph0gLY2Zc9/Xj+dk5Q3lx1Xau+NOb7K3QfEzSuFQOIi3UFSf3544LR7Hsg91c/MclfFymCYyl8agcRFqw80ZnMfvyHNZu38vkWQVsLtkfdCRpJVQOIi3cGYN78MC0kyguPcCke95g/Y6yoCNJK6ByEGkFxvTvwrxrxlJZ5UyZVcDbkZKgI0kLp3IQaSVO6J1O/oxxpCUnMnX2Yt7YsDPoSNKCqRxEWpFjurUnf8Z4sjq344q5S1m4clvQkaSFUjmItDI901N57JpxDO3die89WMhjyzYFHUlaIJWDSCuUkZbMQ1edxMkDuvHj/LeZ8/eNQUeSFkblINJKtU9JYs53cvn28F784tnV3LZwjSbskwbT9BkirVhKUiJ3Th1Np3Yh7l60gd3lldw0cRiJCRZ0NIlzKgeRVi4xwfjV+cPonBbi//62gT37K7l9yiiSkzRwIPVTOYi0AWbGjycMJiMtxK+eW0Pp/kpmXZZDWrIeAqRueuog0oZMP+U4bs0bwT/W7+SSOUsoKT8YdCSJUyoHkTZmyol9+L9Lcli5uZQLZy1me2lF0JEkDqkcRNqgCcN6ct93TySyu5y8e97gg537go4kcUblINJGjR/QjYevHsu+A4eYNLOAVVtKg44kcUTlINKGjeyTwfwZ4wglGhfOLmDpB7uCjiRxQuUg0sYN6N6R/O+NJ7NDCpfdu4RFa3YEHUnigMpBRMjKaMdjM8YxoHsHrv7zMp5cvjnoSBIwlYOIANCtQwqPXD2WnH6due7R5fy54IOgI0mAVA4i8qmOqSHuv3IMXxvcg//35Ep+/9I6zcfURqkcROQzUkOJzLw0m7zsMLe/tJafP72K6moVRFuj786LSC1JiQncNmkE6e1CzP3H++zZX8mtk0YQStTzybZC5SAidUpIMH569hC6tA/xm7+uZW9FJX+4OJvUUGLQ0aQZ6GmAiNTLzLj2jIHcdN4wXl6zg8vvfZPSisqgY0kzUDmIyBFdNrYfv79oNEUf7eaiWYsp3nsg6EjSxBpUDmY2wczeM7P1ZnZ9HetvN7PlsZ+1ZlZSY92tZrbSzFab2Z0WlWZmz5rZmti6m2tsn2Jmj8b2tcTMjmmMOyoiX865I3sz5zu5vL9zH1NmFbBpV3nQkaQJHbEczCwRuBs4ExgKTDWzoTW3cfcfuvsodx8F3AU8HrvueOBkYAQwDDgRODV2td+4+2BgNHCymZ0Zu3wasNvdBwC3A7d8ubsoIo3ltEHdefCqMXxcdoDJMwtYt31v0JGkiTTklcMYYL27b3T3g8A8YOLnbD8VeCT2uwOpQDKQAoSA7e5e7u6LAGK3WQSEY9eZCNwf+z0f+JqZ6ZyGInEip18XHr1mHFXuTJ5VwPJNJUe+krQ4DSmHLGBTjeVI7LJazKwf0B94BcDdC4BFwNbYz0J3X33YdTKAc4CXD9+fux8C9gBdG3Z3RKQ5DOnViQUzxtMpNcTFf1zM6+t2Bh1JGllDyqGuZ+31fSPmIiDf3asAzGwAMIToq4Is4AwzO+XTGzZLIvoq405333g0+zOz6Wa2zMyWFRcXN+BuiEhj6ts1jfwZ4+jbJY0r71vK8+9sDTqSNKKGlEME6FNjOQxsqWfbi/jnkBLA+cBidy9z9zLgeWBsjfWzgXXufkdd+4uVRzpQax5hd5/t7rnunpuZmdmAuyEija17p1QenT6O4eF0vv9wEfPe/CjoSNJIGlIOS4GBZtbfzJKJFsBTh29kZoOAzkBBjYs/Ak41syQzCxF9M3p1bPtfEH3gv+6wm3oK+E7s90nAK67JXUTiVnpaiAemjeGrAzO5/vF3mPnqhqAjSSM4YjnExv2vBRYSfWB/zN1XmtmNZnZujU2nAvMOeyDPBzYA7wArgBXu/rSZhYH/Ifrpp6LYR2Cvil3nXqCrma0HfgTU+uisiMSXtOQk/nh5LueM7M3Nz6/h18+v1oR9LZy1hj9gbm6uL1u2LOgYIm1eVbVzw1Pv8uDij7joxD788vzhJCbow4bxyswK3T23rnWaW0lEGk1ignHTxGF0TkvmrlfWU1Jeye+njiIlSfMxtTSaPkNEGpWZ8e/fHMRPzx7KCyu3ceV9Syk7cCjoWHKUVA4i0iSmfaU/v508ksUbd3HJnCXs3ncw6EhyFFQOItJk8nLCzLw0h9VbS5k8q4Cte/YHHUkaSOUgIk3qG0N78Ocrx7BtTwWT7ilgY3FZ0JGkAVQOItLkxh7blXnTx1JRWcXkmQW8u3lP0JHkCFQOItIshmWlM3/GOFJDiUydvZglGz8OOpJ8DpWDiDSbYzM7MH/GOLp3SuHyuW/y0qrtQUeSeqgcRKRZ9c5ox/wZ4xnUsyPXPFjI40WRoCNJHVQOItLsurRP5uGrx3JS/y786LEVzH39/aAjyWFUDiISiA4pScy94kS+dUIPbnxmFb97ca3mY4ojKgcRCUxqKJG7L85mSm6YO19exw1PraS6WgURDzS3kogEKikxgVvyRpCRlszs1zZSUl7Jb6eMJJSo565BUjmISODMjP8+awid05K55YU1lFZUcs8lObRL1oR9QVE1i0jc+N5px/HrC4bz6tpiLrt3CXv2VwYdqc1SOYhIXJk6pi93X5zNikgJF84qYMfeiqAjtUkqBxGJO2cN78XcK07ko13lTJ5ZwKZd5UFHanNUDiISl746MJOHrjqJkvJK8u55gzXbSoOO1KaoHEQkbo3u25n5M8ZhBlNmFlD44e6gI7UZKgcRiWvH9+hI/ozxdGmfzKVzlvDq2uKgI7UJKgcRiXt9uqQxf8Z4+ndrz1X3L+WZt7cEHanVUzmISIuQ2TGFR6aPZVSfDH7wyFs8tOTDoCO1aioHEWkx0tuF+POVJ3H6oO78zxPvcvei9ZqPqYmoHESkRWmXnMisy3I4b1Rvblv4Hr98drUKoglo+gwRaXFCiQn8bsooMtKSmfP6+5Tsr+TmC4aTpPmYGo3KQURapIQE44ZzhpKRFuKOl9axZ38ld00dTWpI8zE1BtWsiLRYZsZ1Xz+en50zlBdXbee7f1rK3grNx9QYVA4i0uJdcXJ/7rhwFEs/2MXFf1zCx2UHgo7U4qkcRKRVOG90FrMvz2Ht9r1MnlXA5pL9QUdq0VQOItJqnDG4Bw9MO4ni0gNMuucN1u8oCzpSi6VyEJFWZUz/Lsy7ZiyVVc6UWQW8HSkJOlKLpHIQkVbnhN7p5M8YR1pyIlNnL+aNDTuDjtTiqBxEpFU6plt78meMJ6tzO67401IWrtwWdKQWReUgIq1Wz/RUHrtmHEN7deJ7DxYyf9mmoCO1GCoHEWnVMtKSeeiqkzh5QDf+M/9t5vx9Y9CRWgSVg4i0eu1TkpjznVy+PbwXv3h2NbctXKP5mI5A02eISJuQkpTInVNH06ldiLsXbWB3eSU3TRxGYoIFHS0uNeiVg5lNMLP3zGy9mV1fx/rbzWx57GetmZXUWHerma00s9VmdqeZWezyX5rZJjMrO+y2+prZIjN7y8zeNrOzvuydFBEBSEwwfnX+MP7ltON4eMlH/Ou8tzh4qDroWHHpiK8czCwRuBv4BhABlprZU+6+6pNt3P2HNbb/ATA69vt44GRgRGz168CpwN+Ap4E/AOsO2+X/Ao+5+z1mNhR4DjjmC9w3EZFazIwfTxhMRlqIXz23htL9lcy6LIe0ZA2k1NSQVw5jgPXuvtHdDwLzgImfs/1U4JHY7w6kAslAChACtgO4+2J331rH9R3oFPs9HdD5AEWk0U0/5ThunTSCf6zfySVzllBSfjDoSHGlIeWQBdT8/FckdlktZtYP6A+8AuDuBcAiYGvsZ6G7rz7C/n4GXGpmEaKvGn5Qz76mm9kyM1tWXKwTjovI0ZuS24f/uySHlZtLuXDWYraXVgQdKW40pBzqeremvrf5LwLy3b0KwMwGAEOAMNFCOcPMTjnC/qYC97l7GDgLeMDMauV099nunuvuuZmZmQ24GyIitU0Y1pP7vnsikd3lTJr5Bh/s3Bd0pLjQkHKIAH1qLIepf6jnIv45pARwPrDY3cvcvQx4Hhh7hP1NAx6DT195pALdGpBTROQLGT+gGw9fPZayikNMmlnAqi2lQUcKXEPKYSkw0Mz6m1ky0QJ46vCNzGwQ0BkoqHHxR8CpZpZkZiGib0YfaVjpI+BrsdscQrQcNG4kIk1qZJ8M5s8YRyjRuHB2Acs+2BV0pEAdsRzc/RBwLbCQ6AP7Y+6+0sxuNLNza2w6FZjnn/1mST6wAXgHWAGscPen4dOPuEaANDOLmNnPYtf5d+BqM1tB9FXIFa5vq4hIMxjQvSP53xtPZocULr13CYvW7Ag6UmCsNTzu5ubm+rJly4KOISKtxM6yA1zxpzdZs3Uvv50ykomj6vwMTotnZoXunlvXOk2fISJymG4dUnjk6rHk9OvMdY8u54GCD4KO1OxUDiIideiYGuL+K8fwtcE9+OmTK/n9S+va1HxMKgcRkXqkhhKZeWk2edlhbn9pLT9/ehXV1W2jIPR9cRGRz5GUmMBtk0aQ3i7E3H+8T+n+Sm6ZNIJQYut+bq1yEBE5goQE46dnD6FL+xC/+etaSisq+cPF2aSGEoOO1mRad/WJiDQSM+PaMwZy03nDeHnNDi6f+yalFZVBx2oyKgcRkaNw2dh+/P6i0RR9uJupsxezs+xA0JGahMpBROQonTuyN3O+k8vG4n1MnlnApl3lQUdqdCoHEZEv4LRB3XnwqjF8XHaAyTMLWLd9b9CRGpXKQUTkC8rp14VHrxlHlTuTZxWwfFPJka/UQqgcRES+hCG9OrFgxng6pYa4+I+LeX3dzqAjNQqVg4jIl9S3axr5M8bRt0saV963lBfereskly2LykFEpBF075TKo9PHMTyczr88VMS8Nz8KOtKXonIQEWkk6WkhHpg2hq8OzOT6x99h5qsbgo70hakcREQaUVpyEn+8PJdzRvbm5ufX8OvnV7fICfs0fYaISCNLTkrgjgtHkd4uiVmvbmRPeSW/PH84iQkWdLQGUzmIiDSBxATjponD6JyWzF2vrGfP/kruuGgUKUktYz4mDSuJiDQRM+PfvzmIn549lOff3ca0+5ax78ChoGM1iMpBRKSJTftKf347eSQFGz/m4jlL2L3vYNCRjkjlICLSDPJywsy8NIfVW0uZPKuArXv2Bx3pc6kcRESayTeG9uDPV45h254KJt1TwMbisqAj1UvlICLSjMYe25V508dSUVnF5JkFvLt5T9CR6qRyEBFpZsOy0pk/YxypoUSmzl7Mko0fBx2pFpWDiEgAjs3swPwZ4+jeKYXL577JS6u2Bx3pM1QOIiIB6Z3RjvkzxjO4Z0euebCQJ96KBB3pUyoHEZEAdWmfzENXj+Wk/l344aMr+NM/3g86EqByEBEJXIeUJOZecSLfOqEHP396Fb97cW3g8zGpHERE4kBqKJG7L87mwtw+3PnyOm54aiXV1cEVhOZWEhGJE0mJCdycN5yMtBCzXttISXklv50yklBi8z+PVzmIiMQRM+MnZw0hIy2ZW6bCxLcAAAdcSURBVF5YQ2lFJfdckkO75OadsE/DSiIiceh7px3Hry8Yzqtri7ns3iXs2V/ZrPtXOYiIxKmpY/py98XZrIiUcOGsAnbsrWi2fascRETi2FnDezH3ihP5aFc5k2cWsGlXebPsV+UgIhLnvjowk4euOomS8kry7nmD97btbfJ9qhxERFqA0X07M3/GOMxgyqwCCj/c3aT7UzmIiLQQx/foSP6M8XROC3HpnCW8tra4yfbVoHIwswlm9p6ZrTez6+tYf7uZLY/9rDWzkhrrbjWzlWa22szuNDOLXf5LM9tkZrUmNDezKWa2Kna9h7/MHRQRaU36dElj/ozx9O/Wnmn3L+WFd7c2yX6OWA5mlgjcDZwJDAWmmtnQmtu4+w/dfZS7jwLuAh6PXXc8cDIwAhgGnAicGrva08CYOvY3EPgJcLK7nwBc98XumohI65TZMYVHpo/lqwMzCXdOa5J9NORLcGOA9e6+EcDM5gETgVX1bD8VuCH2uwOpQDJgQAjYDuDui2O3d/j1rwbudvfdse12NPC+iIi0GentQsy94sQmu/2GDCtlAZtqLEdil9ViZv2A/sArAO5eACwCtsZ+Frr76iPs73jgeDP7h5ktNrMJ9exrupktM7NlxcVNN+4mItIWNaQcaj21J/qKoC4XAfnuXgVgZgOAIUCYaKGcYWanHGF/ScBA4DSir0LmmFlGrQDus909191zMzMzG3A3RESkoRpSDhGgT43lMLClnm0vAh6psXw+sNjdy9y9DHgeGNuA/T3p7pXu/j7wHtGyEBGRZtKQclgKDDSz/maWTLQAnjp8IzMbBHQGCmpc/BFwqpklmVmI6JvRRxpW+gtweuw2uxEdZtrYgJwiItJIjlgO7n4IuBZYSPSB/TF3X2lmN5rZuTU2nQrM88+eoSIf2AC8A6wAVrj70/DpR1wjQJqZRczsZ7HrLAQ+NrNVRN+v+E93j7+zb4uItGIW9NmGGkNubq4vW7Ys6BgiIi2KmRW6e25d6/QNaRERqUXlICIitbSKYSUzKwY+/IJX7wbsbMQ4jUW5jo5yHb14zaZcR+fL5Orn7nV+F6BVlMOXYWbL6htzC5JyHR3lOnrxmk25jk5T5dKwkoiI1KJyEBGRWlQOMDvoAPVQrqOjXEcvXrMp19Fpklxt/j0HERGpTa8cRESkFpWDiIjU0qbKwcxSzexNM1sROwXpz2OX9zezJWa2zswejU0wGA+57jOz92ucgnVUc+aqkS/RzN4ys2diy4Eer8/JFfjxMrMPzOyd2P6XxS7rYmYvxo7Xi2bWOU5y/czMNtc4XmcFkCvDzPLNbE3sVMLj4uR41ZUr0ONlZoNq7Hu5mZWa2XVNdbzaVDkAB4Az3H0kMAqYYGZjgVuA2919ILAbmBYnuSA68eCo2M/yZs71iX/js7PpBn28PnF4LoiP43V6bP+ffPb8euDl2PF6ObYcD7kg+nf85Hg9F0Cm3wMvuPtgYCTRv2c8HK+6ckGAx8vd36txOuYcoBx4giY6Xm2qHDyqLLYYiv04cAbRGWQB7gfOi5NcgTOzMPBtYE5s2Qj4eNWVK85NJHqcIKDjFY/MrBNwCnAvgLsfdPcSAj5en5MrnnwN2ODuH9JEx6tNlQN8OhSxHNgBvEh0SvGS2NTk8DmnQW3OXO6+JLbql2b2tpndbmYpzZ0LuAP4MVAdW+5KHByvOnJ9Iujj5cBfzazQzKbHLuvh7lsBYv/tHie5AK6NHa+5AQzfHAsUA3+KDQ/OMbP2BH+86ssFwR6vmmqeWK1JjlebKwd3r4q9LAsDY4iexrTWZs2bqnYuMxsG/AQYDJwIdAH+qzkzmdnZwA53L6x5cR2bNuvxqicXBHy8Yk5292zgTOD7duTT4jaXunLdAxxHdChzK/DbZs6UBGQD97j7aGAfwQ251VRfrqCPFwCx9/jOBeY35X7aXDl8IvYy8W9ET1uaYWZJsVWfdxrU5sw1wd23xoacDgB/Ilpmzelk4Fwz+wCYR3Q46Q6CP161cpnZg3FwvHD3LbH/7iA6HjwG2G5mvQBi/90RD7ncfXvsSUk18Eea/3hFgEiNV8n5RB+Ugz5edeaKg+P1iTOBInffHltukuPVpsrBzDLNLCP2ezvg60TfaFoETIpt9h3gyTjItabGH9yIjiO+25y53P0n7h5292OIvox9xd0vIeDjVU+uS4M+XmbW3sw6fvI78M1YhqeIHicI5t9Xnbk+OV4x59P8/762AZsseophiI6jryLg41VfrqCPVw1T+eeQEjTR8Uo68iatSi/gfjNLJFqMj7n7MxY9Jek8M/sF8BaxN6LiINcrZpZJdChnOTCjmXPV578I9njV56GAj1cP4IloN5EEPOzuL5jZUuAxM5tG9Lzqk+Mk1wMW/bivAx8A1zRzLoAfEP27JRM9V/x3if0/EODxqi/XnUEfLzNLA75x2L5vpgmOl6bPEBGRWtrUsJKIiDSMykFERGpROYiISC0qBxERqUXlICIitagcRESkFpWDiIjU8v8BcU64NTYrWmkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, test_means)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] min_child_samples=200 ...........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............... min_child_samples=200, score=0.780, total= 5.4min\n",
      "[CV] min_child_samples=200 ...........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  5.4min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............... min_child_samples=200, score=0.780, total= 5.6min\n",
      "[CV] min_child_samples=200 ...........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed: 11.0min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............... min_child_samples=200, score=0.780, total= 5.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed: 16.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=23, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      categorical_feature=[0, 3, 4, 5, 6, 7, 8,\n",
       "                                                           10, 12, 15, 16, 17,\n",
       "                                                           19, 20],\n",
       "                                      class_weight=None, colsample_bytree=0.3,\n",
       "                                      importance_type='split',\n",
       "                                      is_unbalance=True, learning_rate=0.1,\n",
       "                                      max_depth=10, min_child_sampl...\n",
       "                                      min_split_gain=0.0, n_estimators=500,\n",
       "                                      n_jobs=4, num_leaves=700,\n",
       "                                      objective='binary', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.9,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0, verbosity=5),\n",
       "             iid='warn', n_jobs=None, param_grid={'min_child_samples': [200]},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='roc_auc', verbose=3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':500,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.3,\n",
    "          'verbosity':5\n",
    "         }\n",
    "\n",
    "lg =LGBMClassifier(silent = False, **params)\n",
    "\n",
    "param_leaves =dict(min_child_samples = [200])\n",
    "\n",
    "grid = GridSearchCV(lg, param_leaves, scoring = 'roc_auc', cv = kfold, verbose = 3, refit = False )\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([170.52440476]),\n",
       " 'std_fit_time': array([3.48555684]),\n",
       " 'mean_score_time': array([157.85615166]),\n",
       " 'std_score_time': array([0.69610872]),\n",
       " 'param_min_child_samples': masked_array(data=[200],\n",
       "              mask=[False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'min_child_samples': 200}],\n",
       " 'split0_test_score': array([0.77973534]),\n",
       " 'split1_test_score': array([0.78023403]),\n",
       " 'split2_test_score': array([0.77996781]),\n",
       " 'mean_test_score': array([0.77997906]),\n",
       " 'std_test_score': array([0.00020374]),\n",
       " 'rank_test_score': array([1])}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] colsample_bytree=0.7 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.7, score=0.789, total= 7.3min\n",
      "[CV] colsample_bytree=0.7 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  7.3min remaining:    0.0s\n",
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.7, score=0.790, total= 7.5min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed: 14.7min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] colsample_bytree=0.7 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.7, score=0.790, total= 5.8min\n",
      "[CV] colsample_bytree=0.8 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.8, score=0.791, total= 5.8min\n",
      "[CV] colsample_bytree=0.8 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.8, score=0.791, total= 5.4min\n",
      "[CV] colsample_bytree=0.8 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.8, score=0.790, total= 5.7min\n",
      "[CV] colsample_bytree=0.9 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.9, score=0.793, total= 6.0min\n",
      "[CV] colsample_bytree=0.9 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.9, score=0.793, total= 5.8min\n",
      "[CV] colsample_bytree=0.9 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................ colsample_bytree=0.9, score=0.793, total= 6.0min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed: 55.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=23, shuffle=True),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "                                      categorical_feature=[0, 3, 4, 5, 6, 7, 8,\n",
       "                                                           10, 12, 15, 16, 17,\n",
       "                                                           19, 20],\n",
       "                                      class_weight=None, colsample_bytree=1.0,\n",
       "                                      importance_type='split',\n",
       "                                      is_unbalance=True, learning_rate=0.1,\n",
       "                                      max_depth=10, min_child_sampl...\n",
       "                                      min_split_gain=0.0, n_estimators=500,\n",
       "                                      n_jobs=4, num_leaves=700,\n",
       "                                      objective='binary', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=0.9,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0, verbosity=5),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'colsample_bytree': [0.7, 0.8, 0.9]},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='roc_auc', verbose=3)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':500,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'min_child_samples':10,\n",
    "          # 'colsample_bytree': 0.3,\n",
    "          'verbosity':5\n",
    "         }\n",
    "\n",
    "lg =LGBMClassifier(silent = False, **params)\n",
    "\n",
    "param_leaves =dict(colsample_bytree = [0.7,0.8,0.9])\n",
    "\n",
    "grid = GridSearchCV(lg, param_leaves, scoring = 'roc_auc', cv = kfold, verbose = 3, refit = False )\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78966118, 0.79073358, 0.79303461])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_[ 'mean_test_score' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV9Z3/8deHEMK+g+wBBEFQZAkx6rQulYqiUleCgoBSXEY71RmXTp3qUP21tp06trVaqiwCgopLsS5gqyKjBBKWIIssIoGwBpB9CUk+vz/uiV5JQi4hyU1u3s/H4z5y7vf7Ped8zsm953PP+jV3R0REJFytaAcgIiJVj5KDiIgUoeQgIiJFKDmIiEgRSg4iIlJE7WgHUB5atmzpnTt3jnYYIiLVyuLFi3e5e6vi6mIiOXTu3JmMjIxohyEiUq2YWVZJdTqsJCIiRSg5iIhIEUoOIiJShJKDiIgUoeQgIiJFKDmIiEgRSg4iIlJERMnBzAab2RozW29mjxRT/7SZLQtea81sb1jdU2a2IngNCyt/0cwyzWy5mc0ys4ZBeYKZvRLMa6GZdT79xRQRiS3uzh/+uY5VW/dXyPRLvQnOzOKAZ4FBQDaQbmaz3X1VWJD3h7W/D+gXDA8B+gN9gQRgnpm95+77gfuDv5jZ74F7gV8DdwBfu3s3M0sFngK+SSoiIjXd8fwCHnn9c15fks3h3Hx6tWtc7vOIZM8hGVjv7hvcPReYCQw9SfvhwIxguBcwz93z3P0QkAkMBghLDAbUAwp7HRoKTAmGZwE/CNqIiNR4B4/lcfvkdF5fks1PL+/Ow4N7VMh8IkkO7YHNYe+zg7IizCwR6AJ8GBRlAleaWX0zawlcCnQMaz8J2A70BP544vzcPQ/YB7QoZl7jzCzDzDJycnIiWAwRkept5/6jDPvLAj77cje/uaEPP738LCrqt3MkyaG4OZfUt2gqMMvd8wHcfS7wLvAZob2JBUDeNxNxHwO0A1bz7aGjiObn7hPcPcndk1q1Kva5USIiMWP9zoNc9+fP+GrXIV4YlcTNAzuWPtJpiCQ5ZBP2ax/oAGwtoW0q3x5SAsDdn3T3vu4+iNCGf90J9fnAK8ANJ87PzGoDTYA9EcQpIhKTMjbu4cbnP+NYXj4zx6VwaY/WFT7PSJJDOtDdzLqYWR1CCWD2iY3MrAfQjNDeQWFZnJm1CIb7AH2AuRbSLSg34Brgi2C02cCoYPhG4EN3L2lPRUQkpr2/Yhu3vLCQZvXr8MbdF9GnQ9NKmW+pVyu5e56Z3QvMAeKAie6+0szGAxnuXpgohgMzT9iQxwPzg2Ni+4ERwfRqAVPMrDGhvYlM4O5gnBeBqWa2ntAeQ+ppL6WISDU0+dOv+O+/r6Jfx6a8MGogzRvUqbR5Wyz8KE9KSnL15yAisaKgwHnq/S/4yycb+GGvM3gmtR/16sSV+3zMbLG7JxVXFxOd/YiIxIpjefk8+NpyZmduZWRKIo9f25u4WpV/Nb+Sg4hIFbHvyHHumrqYBRt28/Dgntx1cdcKu1S1NEoOIiJVwLZ9Rxg9MZ0Nuw7y9LDzuK5fh6jGo+QgIhJla7YfYPSkRRw4msfkMclc1K1ltENSchARiabPvtzFnVMXU79OHK/eeUGFPCepLJQcRESiZHbmVv7j1UwSW9Rn8u3JtG9aL9ohfUPJQUSkkrk7L8z/iiffXU1y5+b89bYkmtSPj3ZY36HkICJSifILnCfeWcWkTzcy5Ny2/M/N51E3vvzvYThdSg4iIpXk6PF87n9lGe+t2M7tF3Xh0SFnUysK9zBEQslBRKQS7D2cy49fyiB949c8OuRsxn6va7RDOiklBxGRCpb99WFGT0pn0+7D/OmWflzdp120QyqVkoOISAVauXUfoyelc+x4Pi/dkUxK1yJ9l1VJSg4iIhVk/roc7pq6mCb14pl+94WcdUajaIcUMSUHEZEK8PribB5+fTndWjdk8phk2jSpG+2QTomSg4hIOXJ3/vzxl/x2zhou6taC50YMoHHdqnUPQySUHEREyklefgGPzV7J9IWb+FHfdvzmxvOoUzuSDjerHiUHEZFycCQ3n/tmLOEfq3dy9yVn8uAPe1TZexgioeQgInKadh88xh1TMsjM3sv4ob257YLO0Q7ptCk5iIichqzdhxg1cRHb9h3l+REDuKJ3m2iHVC6UHEREyihz815un5xOvjsv//h8BiQ2j3ZI5SaiMyVmNtjM1pjZejN7pJj6p81sWfBaa2Z7w+qeMrMVwWtYWPn0YJorzGyimcUH5ZeY2b6w6f2iPBZURKQ8ffjFDlInpFE/IY7X774wphIDRLDnYGZxwLPAICAbSDez2e6+qrCNu98f1v4+oF8wPAToD/QFEoB5Zvaeu+8HpgMjgtFeBsYCzwXv57v71ae5bCIiFWLmok38/K0V9GrbmBdHJ9G6UfW6hyESkew5JAPr3X2Du+cCM4GhJ2k/HJgRDPcC5rl7nrsfAjKBwQDu/q4HgEVAdDtMFREphbvz+w/W8sgbn/Mv3Voyc1xKTCYGiCw5tAc2h73PDsqKMLNEoAvwYVCUCVxpZvXNrCVwKdDxhHHigZHA+2HFF5hZppm9Z2a9I1oSEZEKdDy/gIdmLecP/1zHTQM68MKoJBokxO5p20iWrLgLdb2EtqnALHfPB3D3uWY2EPgMyAEWAHknjPNn4BN3nx+8XwIkuvtBM7sKeAvoXiQos3HAOIBOnTpFsBgiImVz6Fge90xfwry1OfzkB925//LumFXfexgiEcmeQzbf/bXfAdhaQttUvj2kBIC7P+nufd19EKFEs66wzsweA1oBD4S13+/uB4Phd4H4YK/jO9x9grsnuXtSq1atIlgMEZFTl3PgGKkT0vi/9bv41fXn8sCgs2I+MUBkew7pQHcz6wJsIZQAbjmxkZn1AJoR2jsoLIsDmrr7bjPrA/QB5gZ1Y4ErgB+4e0HYOG2AHe7uZpZMKIHtLuPyiYiU2Zc5Bxk9aRG7DuTy19sGcFnPM6IdUqUpNTm4e56Z3QvMAeKAie6+0szGAxnuPjtoOhyYGZxgLhQPzA+y7H5ghLsXHlZ6HsgCFgT1b7j7eOBG4G4zywOOAKknTFNEpMItzvqasVPSqWXGzHEpnNexabRDqlQWC9vdpKQkz8jIiHYYIhIj5qzczk9mLKVtk7pMuT2ZxBYNoh1ShTCzxe6eVFxd7J5qFxEpg6kLNvLY7JX06dCUF0cl0aJhQrRDigolBxERoKDA+c2cNTw/70suP7s1fxzen3p14qIdVtQoOYhIjZebV8BDszJ5a9lWbjm/E+Ov7U3tuOrZD0N5UXIQkRpt/9Hj3D1tMZ+u382DV/TgnkvOrBGXqpZGyUFEaqzt+44yetIi1u88yP/cdB43DNBTfAopOYhIjbR2xwFGT1zEviPHmTh6IN8/SzfThlNyEJEaJ23Dbsa9lEFCfByv3nUBvds1iXZIVY6Sg4jUKH9fvpUHXsmkY/N6TLk9mQ7N6kc7pCpJyUFEaowX5m/giXdWM7BzM/56WxJN69eJdkhVlpKDiMS8ggLniXdWM/HTr7jynDY8PawvdeNr7j0MkVByEJGYdvR4Pv/+WibvLN/G6As7819X9yKuli5VLY2Sg4jErH2Hj/PjqRks+moPP7/qbMZ+r4vuYYiQkoOIxKQte48weuIiNu4+xDOpfRnat9gOLKUESg4iEnNWbd3PmMmLOJybz5Tbk7nwzCL9hUkplBxEJKZ8un4Xd05dTMOE2rx21wX0bNM42iFVS0oOIhIz3lq6hQdnZdK1ZUMm3z6Qtk3qRTukakvJQUSqPXfnuXlf8pv315DStTl/GZlEk3rx0Q6rWlNyEJFqLb/AeXz2SqamZXHNee343U19SKitexhOl5KDiFRbR4/n85MZS5m7agd3fr8rDw/uSS3dw1AulBxEpFracyiXsVPSWbp5L49f04vRF3WJdkgxRclBRKqdTbsPM3rSIrL3HuHPt/TnynPbRjukmBNRP3hmNtjM1pjZejN7pJj6p81sWfBaa2Z7w+qeMrMVwWtYWPn0YJorzGyimcUH5WZmfwjmtdzM+pfHgopIbPg8ex/XP/cpuw/l8vLY85UYKkipycHM4oBngSuBXsBwM+sV3sbd73f3vu7eF/gj8EYw7hCgP9AXOB940MwKLzqeDvQEzgXqAWOD8iuB7sFrHPDc6SygiMSOj9bsZNiEBSTUjuP1uy8kqXPzaIcUsyLZc0gG1rv7BnfPBWYCQ0/SfjgwIxjuBcxz9zx3PwRkAoMB3P1dDwCLgML++YYCLwVVaUBTM9NPA5Ea7tX0zYydkkGXlg14854L6da6YbRDimmRJIf2wOaw99lBWRFmlgh0AT4MijKBK82svpm1BC4FOp4wTjwwEnj/VOZnZuPMLMPMMnJyciJYDBGpjtyd//3HWh56fTkXntmCV+68gNaN60Y7rJgXyQnp4q4L8xLapgKz3D0fwN3nmtlA4DMgB1gA5J0wzp+BT9x9/qnMz90nABMAkpKSSopHRKqxvPwCHn1rBTPTN3ND/w78+oZziY+L6FSpnKZI1nI23/213wHYWkLbVL49pASAuz8ZnI8YRGjDv66wzsweA1oBD5RxfiISow7n5vHjlzKYmb6Z+y7rxu9u6qPEUIkiWdPpQHcz62JmdQglgNknNjKzHkAzQnsHhWVxZtYiGO4D9AHmBu/HAlcAw929IGxSs4HbgquWUoB97r6tTEsnItXSroPHSJ2Qxry1OTx53Tn8+w97qB+GSlbqYSV3zzOze4E5QBww0d1Xmtl4IMPdCxPFcGBmcIK5UDwwP/in7gdGuHvhYaXngSxgQVD/hruPB94FrgLWA4eBMae5jCJSjXy16xCjJi5i54GjTBiZxOW9zoh2SDWSfXdbXj0lJSV5RkZGtMMQkdO0dNPX3DEl9F1+cVQS/To1i3JEsc3MFrt7UnF1ukNaRKqED1bt4L4ZSzijcV0mj0mmS8sG0Q6pRlNyEJGom5aWxS/+toJz2zfhxdEDadkwIdoh1XhKDiISNe7O7+au4dmPvuSynq350y39qF9Hm6WqQP8FEYmK3LwCHnljOW8s2cLw5I78cug51NalqlWGkoOIVLoDR49zz/QlzF+3iwcGncV9l3XTpapVjJKDiFSqHfuPMmZSOmt2HOA3N/bh5qSOpY8klU7JQUQqzfqdBxg1MZ2vD+cycfRALj6rVbRDkhIoOYhIpUjfuIexUzKIj6vFq3dewDntm0Q7JDkJJQcRqXDvfb6Nf3tlGR2a1WPKmGQ6Nq8f7ZCkFEoOIlKhJv7fV/zynVX079SMF25LolmDOtEOSSKg5CAiFaKgwPnVe6v56/yvuKL3GTyT2o+68XHRDksipOQgIuXuWF4+//Hact7O3MqoCxL5xTW9iaulS1WrEyUHESlX+44cZ9xLGSz8ag+PXNmTO7/fVfcwVENKDiJSbrbuPcLoSYv4atchnknty9C+xfYoLNWAkoOIlIsvtu9n9MR0Dh3LY8qYZC7s1jLaIclpUHIQkdP22Ze7uPOlxTRIqM2rd13A2W0bRzskOU1KDiJyWv62bAv/8VomXVo2YPKYZNo1rRftkKQcKDmISJm4OxM+2cCv3vuC87s0Z8LIJJrUj492WFJOlBxE5JTlFzi//PsqJn+2kSF92vL7m88jobbuYYglSg4ickqOHs/npzOX8f7K7Yz9ly7851VnU0v3MMSciHrWMLPBZrbGzNab2SPF1D9tZsuC11oz2xtW95SZrQhew8LK7w2m52bWMqz8EjPbFza9X5zuQopI+fj6UC4jXljInFXb+a+re/Ho1b2UGGJUqXsOZhYHPAsMArKBdDOb7e6rCtu4+/1h7e8D+gXDQ4D+QF8gAZhnZu+5+37gU+DvwMfFzHa+u19d1oUSkfK3ec9hRk1aRPbXR/jT8P4M6dM22iFJBYpkzyEZWO/uG9w9F5gJDD1J++HAjGC4FzDP3fPc/RCQCQwGcPel7r6xzJGLSKVZsWUf1z/3GbsOHGPaHecrMdQAkSSH9sDmsPfZQVkRZpYIdAE+DIoygSvNrH5w6OhSIJJuny4ws0wze8/Mepcwr3FmlmFmGTk5ORFMUkTKYt7aHIb9ZQF14mrx+t0XktylebRDkkoQyQnp4g4oegltU4FZ7p4P4O5zzWwg8BmQAywA8kqZ3xIg0d0PmtlVwFtA9yIBuE8AJgAkJSWVFI+InIbXMjbzszc+p/sZjZg8ZiBnNK4b7ZCkkkSy55DNd3/tdwC2ltA2lW8PKQHg7k+6e193H0Qo0aw72czcfb+7HwyG3wXiw09Yi0jFc3f++M91PDhrOSldW/DqnSlKDDVMJMkhHehuZl3MrA6hBDD7xEZm1gNoRmjvoLAszsxaBMN9gD7A3JPNzMzaWPAIRzNLDmLcHdniiMjpyssv4D/fXMH/fLCW6/u1Z+LogTSqq5vbappSDyu5e56Z3QvMAeKAie6+0szGAxnuXpgohgMz3T38EE88MD/Y1u8HRrh7HoCZ/QR4CGgDLDezd919LHAjcLeZ5QFHgNQTpikiFeRwbh73vbyUf36xk3suOZMHr+ihx23XUBYL292kpCTPyMiIdhgi1dqug8e4Y0oGn2fv5b+HnsPIlMRohyQVzMwWu3tScXW6Q1pE2LjrEKMmLWLH/qM8P2IAP+zdJtohSZQpOYjUcMs27+WOyekUuPPyj1Po36lZtEOSKkDJQaQG++fqHdz78lJaNqrDlDHJdG3VMNohSRWh5CBSQ81YtImfv/k557RvwoujBtKqUUK0Q5IqRMlBpIZxd57+YC1/+HA9l/RoxbO39KdBgjYF8l36RIjUIMfzC/jZG58za3E2Nyd14MnrziU+LqKHM0sNo+QgUkMcPJbHPdOX8MnaHH56eXf+7QfddQ+DlEjJQaQG2HngKLdPTmf1tgM8dcO5DBvYKdohSRWn5CAS477MOcioiYvYfTCXF25L4tKeraMdklQDSg4iMSxj4x7GvpRB7VrGK3em0KdD02iHJNWEkoNIjHp/xXb+beZS2jWtx5QxyXRqUT/aIUk1ouQgEoOmfLaRx99eSd+OTXlx1ECaN6gT7ZCkmlFyEIkhBQXOU3O+4C/zNjCo1xn8IbUf9erERTssqYaUHERixLG8fB6atZy/LdvKiJRO/Pe15xBXS5eqStkoOYjEgP1Hj3PnS4tZsGE3Dw3uwd0Xn6l7GOS0KDmIVHPb9h1hzKR01u88yO9vPo/r+3eIdkgSA5QcRKqxtTsOMGriIg4czWPymGT+pbu6W5fyoeQgUk2lbdjNj1/KoF58HK/cmULvdk2iHZLEECUHkWro7cyt/PurmXRqUZ8ptyfTvmm9aIckMUbJQaSaeWH+Bp54ZzXJnZvz19uSaFI/PtohSQyK6Fm9ZjbYzNaY2Xoze6SY+qfNbFnwWmtme8PqnjKzFcFrWFj5vcH03MxahpWbmf0hqFtuZv1PdyFFYkFBgTP+7VU88c5qhpzblpfuSFZikApT6p6DmcUBzwKDgGwg3cxmu/uqwjbufn9Y+/uAfsHwEKA/0BdIAOaZ2Xvuvh/4FPg78PEJs7wS6B68zgeeC/6K1FhHj+fzwKvLePfz7dx+URceHXI2tXQPg1SgSPYckoH17r7B3XOBmcDQk7QfDswIhnsB89w9z90PAZnAYAB3X+ruG4sZfyjwkoekAU3NrG1kiyMSe/YezuW2Fxfx7ufbeXTI2fziml5KDFLhIkkO7YHNYe+zg7IizCwR6AJ8GBRlAleaWf3g0NGlQMfymJ+ZjTOzDDPLyMnJiWAxRKqf7K8Pc+PzC1i2eS9/HN6Psd/rGu2QpIaI5IR0cT9RvIS2qcAsd88HcPe5ZjYQ+AzIARYAeeUxP3efAEwASEpKKikekWpr5dZ9jJmUzpHj+Uy5PZkLzmwR7ZCkBolkzyGb7/7a7wBsLaFtKt8eUgLA3Z90977uPojQhn9dOc5PJCbNX5fDsL+kEVfLeP3uC5UYpNJFkhzSge5m1sXM6hBKALNPbGRmPYBmhPYOCsvizKxFMNwH6APMLWV+s4HbgquWUoB97r4toqURiQFvLMlmzKR0OjSrx5v3XMRZZzSKdkhSA5V6WMnd88zsXmAOEAdMdPeVZjYeyHD3wkQxHJjp7uGHeOKB+cEDwPYDI9w9D8DMfgI8BLQBlpvZu+4+FngXuApYDxwGxpTDcopUee7Onz/+kt/OWcOFZ7bg+ZEDaFxXl6pKdNh3t+XVU1JSkmdkZEQ7DJEyyy9wHpu9gmlpmxjatx2/vfE86tSO6DYkkTIzs8XunlRcne6QFomyI7n53DdjKf9YvYO7Lj6Th67ooUtVJeqUHESiaM+hXO6Yks6yzXsZP7Q3t13QOdohiQBKDiJRk7X7EKMnpbN17xGeu3UAg89pE+2QRL6h5CASBZmb93LHlHTyCpyXf3w+AxKbRzskke9QchCpZB99sZN7pi+hRcM6TLk9mTNbNYx2SCJFKDmIVKJX0jfxn2+u4Oy2jZg4eiCtG9WNdkgixVJyEKkE7s7//mMdz/xzHd8/qxV/vrU/DRP09ZOqS59OkQp2PL+AR99cwSsZm7lpQAf+3/XnEh+nexikalNyEKlAh47l8a8vL+HjNTn85Afduf/y7gRPDBCp0pQcRCpAQYHzybocfjtnDV9sP8Cvrj+X4cmdoh2WSMSUHETK0Z5DubyWsZnpCzexac9hWjZM4K+3DeCynmdEOzSRU6LkIHKa3J2lm/cybUEWf/98G7l5BSR3ac6DV/Tgit5t9IwkqZaUHETK6HBuHn9btpVpaVms3Lqfhgm1SR3YkVvPT6RHGz1mW6o3JQeRU7R+50GmpWXx+pJsDhzNo2ebRjzxo3P4Ub/2ujxVYoY+ySIROJ5fwAerdjB1QRYLNuwmPs646ty2jExJZEBiM12BJDFHyUHkJLbtO8KMRZuZuWgTOw8co33Tejw0uAc3J3WkZcOEaIcnUmGUHERO4O58un4309Ky+GD1DgrcueSsVvwqJZFLerQmTn0tSA2g5CAS2Hf4OLOWZDM9LYsNuw7RrH48Y7/XhVuTE+nUon60wxOpVEoOUuN9nr2PqWkbmZ25laPHC+jfqSlPDzuPK89pS934uGiHJxIVSg5SIx09ns/bmVuZtnATmZv3Ui8+juv6dWBESid6t2sS7fBEok7JQWqUjbsOMX1hFq9mZLPvyHG6tW7I49f04voBHWhcNz7a4YlUGRElBzMbDDwDxAEvuPuvT6h/Grg0eFsfaO3uTYO6p4AhQd0v3f2VoLwLMBNoDiwBRrp7rpmNBn4LbAnG+ZO7v1C2xROBvPwCPvxiJ1PTspi/bhe1axlX9G7DiJREUro212WoIsUoNTmYWRzwLDAIyAbSzWy2u68qbOPu94e1vw/oFwwPAfoDfYEEYJ6Zvefu+4GngKfdfaaZPQ/cATwXTOYVd7+3PBZQaq6dB47yyqLNzFi0ia37jtKmcV0eGHQWqQM70rqxOtkROZlI9hySgfXuvgHAzGYCQ4FVJbQfDjwWDPcC5rl7HpBnZpnAYDN7DbgMuCVoNwV4nG+Tg0iZuDsLv9rDtLQs3l+xnbwC53vdW/KLa3pz+dmtqa1+FEQiEklyaA9sDnufDZxfXEMzSwS6AB8GRZnAY2b2e0KHmy4llFRaAHuDpFE4zfZhk7rBzL4PrAXud/fw+RfOaxwwDqBTJz0KuaY7cPQ4by7dwtQFWazbeZDGdWsz6sLO3Hp+J7qqj2aRUxZJcijugKyX0DYVmOXu+QDuPtfMBgKfATnAAiCvlGm+Dcxw92NmdhehvYrLijR2nwBMAEhKSiopHolxq7buZ9rCLN5auoXDufn06dCE39zYh2v6tKNeHV2GKlJWkSSHbKBj2PsOwNYS2qYC/xpe4O5PAk8CmNnLwDpgF9DUzGoHew/fTNPdd4eN/ldC5yZEvnEsL5/3Pt/OtLQsMrK+JqF2La49rx0jUhI5r2PTaIcnEhMiSQ7pQPfg6qIthBLALSc2MrMeQDNCeweFZXFAU3ffbWZ9gD7AXHd3M/sIuJHQFUujgL8F47R1923BJK4FVpd14SS2bN5zmJcXbeLV9M3sPpRL5xb1eXTI2dw4oANN69eJdngiMaXU5ODueWZ2LzCH0KWsE919pZmNBzLcfXbQdDgw093DD/HEA/ODSwX3AyPCzjM8DMw0syeApcCLQflPzOxaQoef9gCjT2cBpXrLL3A+WZvD1LQsPlqzEwMuP/sMRl6QyEVntqSWnnMkUiHsu9vy6ikpKckzMjKiHYaUo90Hj/FqRjYvL8pi854jtGqUwPCBHUlN7kS7pvWiHZ5ITDCzxe6eVFyd7pCWKsPdWbLpa6albeKd5dvIzS8gpWtzHh7ckx/2UnebIpVJyUGi7tCxUHebU9OyWL1tP40SajM8uSMjUhLpfoa62xSJBiUHiZp1Ow4wLS2LN5Zs4cCxPM5u25j/d925DO3bjgbqblMkqvQNlEqVm1fA3FWhy1DTNuyhTlwthvRpy4iURPp3aqrnHIlUEUoOUim27TvCjIWbmJG+mZwDx+jQrB4PD+7JzUkdaKHuNkWqHCUHqTAFBc6nX+5i6oIs/rF6Bw5c2qM1I1MS+f5ZrdTdpkgVpuQg5W7v4VxmLc5m+sJNfLXrEM0b1OHOi8/kluROdGyu7jZFqgMlByk3mZv3Mi0ti9mZWzmWV0BSYjN+enl3Bp/ThoTaes6RSHWi5CCn5UhuPm8v38q0tCyWZ++jfp04bhjQgRHnJ9KrXeNohyciZaTkIGWyIecg0xdu4rWMzew/mkf31g0ZP7Q31/VrTyN1tylS7Sk5SMTy8gv4x+qdTF/4bXebg89pw8iURJK7qLtNkVii5CCl2rn/KDPTN/Pywk1s33+Udk3q8u+DzmJYckdaN1J3myKxSMlBiuXupG0Idbc5Z+W33W2OH9qby3qqu02RWKfkIN+x/+hx3liczbSFm1i/8yBN6sUz5qLO3HJ+Il1aNoh2eCJSSZQcBICVW/cxLW0Tby3dwpHj+ZzXsSm/u+k8ru7TlrrxugxVpKZRcqjBjh7P570V25i6IIslm/ZSN/7b7jb7dFB3myI1mZJDDbRp9043yU4AAA3hSURBVGGmL8ri1fTNfH34OF1bNuC/ru7Fjf070KS+LkMVESWHGiO/wPl4zU6mpWXx8docapkxKOhu88IzW+gyVBH5DiWHGLfr4DFezdjM9LRNbNl7hNaNErjvsu4MT+5I2ybqblNEiqfkEIPcncVZXzM1LYt3P9/G8Xzngq4t+PmQsxnU6wzidRmqiJQiouRgZoOBZ4A44AV3//UJ9U8DlwZv6wOt3b1pUPcUMCSo+6W7vxKUdwFmAs2BJcBId881swTgJWAAsBsY5u4by7yENcjBY3m8tXQL09Ky+GL7ARol1ObW8xMZkdKJbq3V3aaIRK7U5GBmccCzwCAgG0g3s9nuvqqwjbvfH9b+PqBfMDwE6A/0BRKAeWb2nrvvB54Cnnb3mWb2PHAH8Fzw92t372ZmqUG7YeWytDFqbVh3mweP5dGrbWN+dX2ou836dbRzKCKnLpItRzKw3t03AJjZTGAosKqE9sOBx4LhXsA8d88D8swsExhsZq8BlwG3BO2mAI8TSg5Dg2GAWcCfzMzc3U9huWJebl4Bc1ZuZ2paFou+CnW3eXWftoy4IJF+HdXdpoicnkiSQ3tgc9j7bOD84hqaWSLQBfgwKMoEHjOz3xM63HQpoaTSAtgbJI3CabY/cX7unmdm+4L2u06Y1zhgHECnTp0iWIzYsGVvqLvNmemb2XXwGB2b1+NnV/bkpqSONG9QJ9rhiUiMiCQ5FPcTtKRf8anALHfPB3D3uWY2EPgMyAEWAHmlTDOi+bn7BGACQFJSUkzvVRQUOPPXh7rb/PCLUHebP+jZmltTErm4eytqqbtNESlnkSSHbKBj2PsOwNYS2qYC/xpe4O5PAk8CmNnLwDpCewFNzax2sPcQPs3C+WWbWW2gCbAnoqWJMV8fCnW3OW1hFlm7D9OiQR3uuvhMhqu7TRGpYJEkh3Sge3B10RZCCeCWExuZWQ+gGaG9g8KyOKCpu+82sz5AH2Cuu7uZfQTcSOiKpVHA34LRZgfvFwT1H9ak8w3uTmb2PqYuyOLt5VvJzStgYOdmPDDoLHW3KSKVptTkEBz3vxeYQ+hS1onuvtLMxgMZ7j47aDocmHnChjwemB+cHN0PjAg7z/AwMNPMngCWAi8G5S8CU81sPaE9htTTWsJq4khuPrMztzA1LYsVW/bToE4cNyd1YERKIj3bqLtNEalcFgs/ypOSkjwjIyPaYZTJlzkHmZaWxeuLs9l/NI+zzmjIyJREfqTuNkWkgpnZYndPKq5OF8FHQai7zR1MTcvi0/W7iY8zBp/TlpEpiQzs3EyXoYpI1Ck5VKId+48yY9EmZizaxI79x2jXpC4PXtGDm5M60qpRQrTDExH5hpJDBXN3Fny5m6lpWcxdtYP8Aufis1rxxI8Suaxna+J0GaqIVEFKDhVk35HjvLEkm2lpWXyZc4im9eO541+6cEtyJzqru00RqeKUHMrZii37mJaWxd+WbeXI8Xz6dmzK/9x0HkPU3aaIVCNKDuXg6PF83lm+jalpWSzbHOpu80d92zMiJZFz2jeJdngiIqdMyeE0ZO0+xPSFm3gtI+hus1UDfnF1L24Y0IEm9XQZqohUX0oOpyi/wPnoi51MTcti3toc4moZP+x1BiNTErlA3W2KSIxQcohQzoFQd5svLwx1t3lG4wR+enl3Ugd2ok2TutEOT0SkXCk5nIS7k74x1N3m+ytC3W1e1K0Fjw45m8vV3aaIxDAlh2IcOHo86G5zE2t2HKBR3dqMSEnk1vMT6da6YbTDExGpcEoOYb7Yvp9paVm8uWQLh3Lz6d2uMU/dcC7XnKfuNkWkZqnxW7xjefm8v2I709KySN/4NXVq1+KaPu0YkdKJvupuU0RqqBqdHD78YgcPzVrOroO5JLaoz39e1ZObBnSkmbrbFJEarkYnh07NG9C3Y1NGXtCZ73Vrqe42RUQCNTo5dGvdkBdGDYx2GCIiVY6uxRQRkSKUHEREpAglBxERKULJQUREiogoOZjZYDNbY2brzeyRYuqfNrNlwWutme0Nq/uNma00s9Vm9gcLbhwws2Fmtjyo+01Y+9FmlhM2vbHlsaAiIhK5Uq9WMrM44FlgEJANpJvZbHdfVdjG3e8Pa38f0C8YvhC4COgTVP8fcLGZfQ78Fhjg7jlmNsXMfuDu/wzaveLu957+4omISFlEsueQDKx39w3ungvMBIaepP1wYEYw7EBdoA6QAMQDO4CuwFp3zwna/QO44dTDFxGRihBJcmgPbA57nx2UFWFmiUAX4EMAd18AfARsC15z3H01sB7oaWadzaw28COgY9ikbggOOc0ys44Uw8zGmVmGmWXk5OQU10RERMookpvgirtt2EtomwrMcvd8ADPrBpwNdAjqPzCz77v7J2Z2N/AKUAB8RmhvAuBtYIa7HzOzu4ApwGVFAnCfAEwI5pNjZlkRLEtxWgK7yjhuRaqqcUHVjU1xnRrFdWpiMa7EkioiSQ7ZfPdXfQdgawltU4F/DXt/HZDm7gcBzOw9IAX4xN3fJpQIMLNxQD6Au+8OG/+vwFOlBejurSJYjmKZWYa7J5V1/IpSVeOCqhub4jo1iuvU1LS4IjmslA50N7MuZlaHUAKYXUyAPYBmwIKw4k2ETkDXNrN44GJgddC+dfC3GXAP8ELwvm3Y+NcWthcRkcpT6p6Du+eZ2b3AHCAOmOjuK81sPJDh7oWJYjgw093DDznNInRI6HNCh6LeD/YYAJ4xs/OC4fHuvjYY/omZXQvkAXuA0WVfPBERKYuIHrzn7u8C755Q9osT3j9ezHj5wJ0lTHN4CeU/A34WSVzlZEIlzutUVNW4oOrGprhOjeI6NTUqLvvuD30RERE9PkNERIqh5CAiIkXEXHI4zedAjTKzdcFrVFj5ADP7PJjmN8+Hqoy4zKyvmS0InkG13MyGhY0z2cy+Chuvb2XFFdTlh9XNDivvYmYLg/X4SnCVW6XEZWaXhpUvM7OjZvajoK4y1lcnM/vIzJYG/6+rwup+Foy3xsyuiHSaFRmXmQ0ys8XB53uxmV0WNs7HwTQL11frSoyrs5kdCZv382HjVMb3saS4bj3h81VQ+DmqpPWVaGb/DGL62Mw6hNWV7/bL3WPmRehqqi8J3VBXB8gEep2k/X2Err4CaA5sCP42C4abBXWLgAsI3RD4HnBlJcZ1FtA9GG5H6E7zpsH7ycCN0VhfwfuDJbR7FUgNhp8H7q7MuMLKmxO64q1+Za0vQicH7w6GewEbw4YzCT1GpkswnbhTXdYKiKsf0C4YPgfYEjbOx0BSlNZXZ2BFCdOt8O9jSXGd0OZcYEMlr6/XgFHB8GXA1LDPerluv2Jtz+F0ngN1BfCBu+9x96+BD4DBFrrvorG7L/DQmn6J0OM+KiUud1/r7uuC4a3ATqDMN/2VV1wlCX6VXEboMmYI3eFeaevrBDcC77n74VOc/+nE5UDjYLgJ394wOpTQpd7H3P0rQo+QSY5wmhUWl7svDT5XACuBumaWcIrzL/e4SlKJ38dI4ir1+1ABcfUCCh9Q+lFYfblvv2ItOZT5OVAnGbd9MFzqNCsorvC6ZEK/KL4MK34y2MV8ugxf6tONq66Fnm+VVnjoBmgB7HX3vNKmWYFxFUql6Je3otfX48AIM8smdPn3faWMG/GyVlBc4W4Alrr7sbCyScEhkv8qw+Gb042rS3BYZ56ZfS9smpXxfTxZXIWGUfTzVdHrK5NvH1J6HdDIzFqcZNwyr69YSw5lfg7UScY9lWlWRFyhCYR+AUwFxrh7QVD8M6AnMJDQ7uTDlRxXJw/dtn8L8L9mduYpTrOi4ipcX+cSunmzUGWsr+HAZHfvAFwFTDWzWicZt7LWV0lxhSZg1pvQo2rC70u61d3PBb4XvEZWYlzbCH2++gEPAC+bWeMIp1mRcYUmYHY+cNjdV4SNUxnr6z8IPXViKaEnTmwhdMNwuX++Yi05nOpzoMKzfknjZvPtgwNLm2ZFxEXwpXgHeNTd0wrL3X2bhxwDJhHaLa20uAoPR7j7BkLHW/sRegBYUws9bbe0aVZIXIGbgTfd/XhYvJWxvu4gdM4FDz2VuC6hB6Od7PMV6bJWRFwEJzXfBG5z92/2St19S/D3APAylbi+gsNvu4PyxYT2ls+i8r6PJa6vQHHfhwpfX+6+1d2vD5Lmz4OyfScZt+zrq6wnT6rii9Ad3xsIHWYoPKHTu5h2PYCNBDcB+rcndL4idDKnWTDcPKhLJ/TAwMITOldVYlx1CB1j/Gkx7dsGfw34X+DXlRhXMyAhGG4JrCM4eUbopFn4Cel7KiuusLo04NLKXl/B52N0MHw2oS+iAb357gnpDYROQEa0rBUYV9Og/Q3FTLNlMBxP6BzSXZUYVysgLijvSugXcqV9H0uKK3hfi9BGt2sU1ldLoFYw/CShRw9BBWy/Ig68urwI7QKuJfRL4+dB2Xjg2rA2j1PMhgG4ndCJwvWEDt8UlicBK4Jp/oliNkYVFRcwAjgOLAt79Q3qPiT03KoVwDSgYSXGdWEw78zg7x1hdV0JXSGxnlCiSKjk/2NnQhuTWieUV/j6InTC8NNgvSwDfhg27s+D8dYQdsVIcdOsrLiAR4FDJ3y+WgMNgMXAckInqp8h2FhXUlw3BPPNBJYA11Tm97GU/+MlhJ42HT69ylpfNxL6IbaW0MNKE8LGLdftlx6fISIiRcTaOQcRESkHSg4iIlKEkoOIiBSh5CAiIkUoOYiISBFKDiIiUoSSg4iIFPH/AaRsB/VVIc6cAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "\n",
    "plt.plot([0.7,0.8,0.9], test_means)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## n_estimators "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 3000\n",
      "best cv score: 0.8132423156645844\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          # 'n_estimators':500,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.9,\n",
    "          'verbosity':5\n",
    "         }\n",
    "\n",
    "n_estimators_1 = get_n_estimators(params , X_train , y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "               categorical_feature=[0, 3, 4, 5, 6, 7, 8, 10, 12, 15, 16, 17, 19,\n",
       "                                    20],\n",
       "               class_weight=None, colsample_bytree=0.9, importance_type='split',\n",
       "               is_unbalance=True, learning_rate=0.1, max_depth=10,\n",
       "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "               n_estimators=3000, n_jobs=4, num_leaves=700, objective='binary',\n",
       "               random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=False,\n",
       "               subsample=0.9, subsample_for_bin=200000, subsample_freq=0,\n",
       "               verbosity=5)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.9,\n",
    "          'verbosity':5\n",
    "         }\n",
    "lg =LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>song_id_counts</td>\n",
       "      <td>172998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>song_length_s</td>\n",
       "      <td>144121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>msno_counts</td>\n",
       "      <td>137044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>use_days</td>\n",
       "      <td>125707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>artist_name_counts</td>\n",
       "      <td>118077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bd</td>\n",
       "      <td>61700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>genre_ids_counts</td>\n",
       "      <td>54353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>artist_name</td>\n",
       "      <td>45090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gender</td>\n",
       "      <td>33100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>msno_id</td>\n",
       "      <td>26123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>source_screen_name</td>\n",
       "      <td>20038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>source_type</td>\n",
       "      <td>13628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sid</td>\n",
       "      <td>12474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>expiration_date_Ym</td>\n",
       "      <td>11035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>city</td>\n",
       "      <td>9995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>genre_ids</td>\n",
       "      <td>9932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>language</td>\n",
       "      <td>3374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>source_system_tab</td>\n",
       "      <td>3338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>expiration_date_month</td>\n",
       "      <td>2197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>registered_via</td>\n",
       "      <td>885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>expiration_date_year</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  columns  importance\n",
       "13         song_id_counts      172998\n",
       "14          song_length_s      144121\n",
       "11            msno_counts      137044\n",
       "18               use_days      125707\n",
       "1      artist_name_counts      118077\n",
       "2                      bd       61700\n",
       "9        genre_ids_counts       54353\n",
       "0             artist_name       45090\n",
       "7                  gender       33100\n",
       "19                msno_id       26123\n",
       "15     source_screen_name       20038\n",
       "17            source_type       13628\n",
       "20                    sid       12474\n",
       "4      expiration_date_Ym       11035\n",
       "3                    city        9995\n",
       "8               genre_ids        9932\n",
       "10               language        3374\n",
       "16      source_system_tab        3338\n",
       "5   expiration_date_month        2197\n",
       "12         registered_via         885\n",
       "6    expiration_date_year          53"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"columns\":list(X_train.columns), \"importance\":list(lg.feature_importances_)})\n",
    "df = df.sort_values(by=['importance'],ascending=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "X_test = csr_matrix(X_test)\n",
    "y_test_pred = lg.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 2)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = [x[1] for x in y_test_pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.387595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.388619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.142298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.063313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.031516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    target\n",
       "0   0  0.387595\n",
       "1   1  0.388619\n",
       "2   2  0.142298\n",
       "3   3  0.063313\n",
       "4   4  0.031516"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_LGBM = pd.DataFrame({'id':test['id'],'target':y_pred})\n",
    "result_LGBM.id = result_LGBM.id.astype('int64')\n",
    "result_LGBM.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2556790"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result_LGBM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result_LGBM.to_csv('result_gbdt.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "               categorical_feature=[0, 3, 4, 5, 6, 7, 8, 10, 12, 15, 16, 17, 19,\n",
       "                                    20],\n",
       "               class_weight=None, colsample_bytree=0.9, importance_type='split',\n",
       "               is_unbalance=True, learning_rate=0.1, max_depth=10,\n",
       "               min_child_samples=200, min_child_weight=0.001,\n",
       "               min_split_gain=0.0, n_estimators=3000, n_jobs=2, num_leaves=700,\n",
       "               objective='binary', random_state=None, reg_alpha=0.0,\n",
       "               reg_lambda=0.0, silent=False, subsample=0.9,\n",
       "               subsample_for_bin=200000, subsample_freq=0, verbosity=5)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 2,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':3000,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.9,\n",
    "          'verbosity':5,\n",
    "          'min_child_samples':200\n",
    "         }\n",
    "lg =LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.456651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.325782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.193065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.070255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.044401</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    target\n",
       "0   0  0.456651\n",
       "1   1  0.325782\n",
       "2   2  0.193065\n",
       "3   3  0.070255\n",
       "4   4  0.044401"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = lg.predict_proba(X_test)\n",
    "y_pred = [x[1] for x in y_test_pred]\n",
    "result_LGBM = pd.DataFrame({'id':test['id'],'target':y_pred})\n",
    "result_LGBM.id = result_LGBM.id.astype('int64')\n",
    "result_LGBM.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result_LGBM.to_csv('result_gbdt2.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_freq=1, boosting_type='dart',\n",
       "               categorical_feature=[0, 3, 4, 5, 6, 7, 8, 10, 12, 15, 16, 17, 19,\n",
       "                                    20],\n",
       "               class_weight=None, colsample_bytree=0.9, importance_type='split',\n",
       "               is_unbalance=True, learning_rate=0.1, max_depth=10,\n",
       "               min_child_samples=200, min_child_weight=0.001,\n",
       "               min_split_gain=0.0, n_estimators=5000, n_jobs=4, num_leaves=700,\n",
       "               objective='binary', random_state=None, reg_alpha=0.0,\n",
       "               reg_lambda=0.0, silent=False, subsample=0.9,\n",
       "               subsample_for_bin=200000, subsample_freq=0, verbosity=5)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'dart',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':5000,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.9,\n",
    "          'verbosity':5,\n",
    "          'min_child_samples':200\n",
    "\n",
    "         }\n",
    "lg =LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>song_id_counts</td>\n",
       "      <td>114547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>artist_name_counts</td>\n",
       "      <td>111576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>msno_counts</td>\n",
       "      <td>110566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>artist_name</td>\n",
       "      <td>107510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>msno_id</td>\n",
       "      <td>84659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>use_days</td>\n",
       "      <td>75972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>song_length_s</td>\n",
       "      <td>73711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sid</td>\n",
       "      <td>44544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>genre_ids_counts</td>\n",
       "      <td>39384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bd</td>\n",
       "      <td>30096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>source_screen_name</td>\n",
       "      <td>27769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gender</td>\n",
       "      <td>22582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>expiration_date_Ym</td>\n",
       "      <td>20859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>source_type</td>\n",
       "      <td>17993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>city</td>\n",
       "      <td>17366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>genre_ids</td>\n",
       "      <td>14791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>source_system_tab</td>\n",
       "      <td>4995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>language</td>\n",
       "      <td>4137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>expiration_date_month</td>\n",
       "      <td>3684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>registered_via</td>\n",
       "      <td>2125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>expiration_date_year</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  columns  importance\n",
       "13         song_id_counts      114547\n",
       "1      artist_name_counts      111576\n",
       "11            msno_counts      110566\n",
       "0             artist_name      107510\n",
       "19                msno_id       84659\n",
       "18               use_days       75972\n",
       "14          song_length_s       73711\n",
       "20                    sid       44544\n",
       "9        genre_ids_counts       39384\n",
       "2                      bd       30096\n",
       "15     source_screen_name       27769\n",
       "7                  gender       22582\n",
       "4      expiration_date_Ym       20859\n",
       "17            source_type       17993\n",
       "3                    city       17366\n",
       "8               genre_ids       14791\n",
       "16      source_system_tab        4995\n",
       "10               language        4137\n",
       "5   expiration_date_month        3684\n",
       "12         registered_via        2125\n",
       "6    expiration_date_year         127"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"columns\":list(X_train.columns), \"importance\":list(lg.feature_importances_)})\n",
    "df = df.sort_values(by=['importance'],ascending=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.402246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.414657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.178202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.100144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.104373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    target\n",
       "0   0  0.402246\n",
       "1   1  0.414657\n",
       "2   2  0.178202\n",
       "3   3  0.100144\n",
       "4   4  0.104373"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = lg.predict_proba(X_test)\n",
    "y_pred = [x[1] for x in y_test_pred]\n",
    "result_LGBM = pd.DataFrame({'id':test['id'],'target':y_pred})\n",
    "result_LGBM.id = result_LGBM.id.astype('int64')\n",
    "result_LGBM.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import _pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result_LGBM.to_csv('result_dart.csv',index = False)\n",
    "_pickle.dump(lg,open('dart.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python37\\lib\\site-packages\\lightgbm\\basic.py:741: UserWarning: categorical_feature keyword has been found in `params` and will be ignored.\n",
      "Please use categorical_feature argument of the Dataset constructor to pass this parameter.\n",
      "  .format(key))\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':5000,\n",
    "          'num_leaves': 700,\n",
    "          'max_depth': 10,\n",
    "          'subsample': 0.9,\n",
    "          'bagging_freq': 1,\n",
    "          'colsample_bytree': 0.9,\n",
    "          'verbosity':5,\n",
    "          'min_child_samples':200\n",
    "         }\n",
    "lg2 =LGBMClassifier(silent=False,  **params)\n",
    "lg2.fit(X_train, y_train)\n",
    "_pickle.dump(lg2,open('gbdt.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_freq=1, boosting_type='gbdt',\n",
       "               categorical_feature=[0, 3, 4, 5, 6, 7, 8, 10, 12, 15, 16, 17, 19,\n",
       "                                    20],\n",
       "               class_weight=None, colsample_bytree=0.9, importance_type='split',\n",
       "               is_unbalance=True, learning_rate=0.1, max_depth=10,\n",
       "               min_child_samples=200, min_child_weight=0.001,\n",
       "               min_split_gain=0.0, n_estimators=5000, n_jobs=4, num_leaves=700,\n",
       "               objective='binary', random_state=None, reg_alpha=0.0,\n",
       "               reg_lambda=0.0, silent=False, subsample=0.9,\n",
       "               subsample_for_bin=200000, subsample_freq=0, verbosity=5)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':2000,\n",
    "          'num_leaves': 650,\n",
    "          'max_depth': 10,\n",
    "          'min_child_samples': 25,\n",
    "          'colsample_bytree': 0.3,\n",
    "          'verbosity':5\n",
    "         }\n",
    "lg3 =LGBMClassifier(silent=False,  **params)\n",
    "lg3.fit(X_train, y_train)\n",
    "y_test_pred = lg3.predict_proba(X_test)\n",
    "y_pred = [x[1] for x in y_test_pred]\n",
    "result_LGBM = pd.DataFrame({'id':test['id'],'target':y_pred})\n",
    "result_LGBM.id = result_LGBM.id.astype('int64')\n",
    "result_LGBM.to_csv('result_goss_depth10.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True,\n",
    "          'categorical_feature': [0,3,4,5,6,7,8,10,12,15,16,17,19,20],\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':5000,\n",
    "          'num_leaves': 650,\n",
    "          'max_depth': 20,\n",
    "          'colsample_bytree': 0.3,\n",
    "          'verbosity':5,\n",
    "          'min_child_samples':200\n",
    "         }\n",
    "lg3 =LGBMClassifier(silent=False,  **params)\n",
    "lg3.fit(X_train, y_train)\n",
    "_pickle.dump(lg3,open('goss.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
